From c2643317c9c36507c94f8ac9583e03f726ccca35 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 6 Jan 2020 17:11:55 +0800 Subject: [PATCH 01/14] Change milvus helm version to master (#929) * specify multiple urls on sqlite_orm download stage * fix bug * fix bug * specify multiple urls on opentracing download stage * fix bug * specify multiple urls on download stage * delete jfrog cache * print jenkins enviroment variables * print jenkins enviroment variables * fix check_ccache.sh bug * debug * Update Jenkinfile * Add build enviroment resource limit on Jenkins CI * remove Jfrog cache build option * Custom defined name for the upload on Codecov * change milvus helm version to master --- ci/jenkins/step/deploySingle2Dev.groovy | 7 ++----- ci/jenkins/step/singleDevNightlyTest.groovy | 8 +++----- ci/jenkins/step/singleDevTest.groovy | 10 ++++------ 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index 3a5854b58d..657c0146ba 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -2,9 +2,6 @@ sh 'helm version' sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) - dir ("milvus") { - sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." - } + checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } - diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 823cc25660..7244ddf300 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,7 +1,7 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } // mysql database backend test load "ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" @@ -12,11 +12,9 @@ timeout(time: 90, unit: 'MINUTES') { } } dir ("milvus-helm") { - dir ("milvus") { - sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." - } + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 0ad9e369bf..cad156ed94 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,7 +1,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } // mysql database backend test @@ -9,15 +9,13 @@ timeout(time: 60, unit: 'MINUTES') { // if (!fileExists('milvus-helm')) { // dir ("milvus-helm") { - // checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) + // checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) // } // } // dir ("milvus-helm") { - // dir ("milvus") { - // sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." - // } + // sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" // } } From 28e61ee4ec25c05a089ba4a0bb55f7cb0c06150d Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Mon, 6 Jan 2020 17:40:28 +0800 Subject: [PATCH 02/14] [skip ci] Change Milvus c++ standard to c++17 (#932) * [skip ci]#668 - Update badge of README * Fix README.md * Fix README.md * Merge remote-tracking branch 'upstream/master' * #910 Change c++ standard to c++17 * Remove unused headers * #910 Change Milvus c++ standard to c++17 * #910 Change Milvus c++ standard to c++17 --- install.md => INSTALL.md | 1 + 1 file changed, 1 insertion(+) rename install.md => INSTALL.md (98%) diff --git a/install.md b/INSTALL.md similarity index 98% rename from install.md rename to INSTALL.md index 4051e4deb4..3eb544597e 100644 --- a/install.md +++ b/INSTALL.md @@ -13,6 +13,7 @@ If you encounter any problems/issues compiling Milvus from source, please refer If your operating system is not Ubuntu 18.04 or higher, we recommend you to pull a [docker image of Ubuntu 18.04](https://docs.docker.com/install/linux/docker-ce/ubuntu/) as your compilation environment. +- GCC 7.0 or higher to support C++17 - CMake 3.12 or higher ##### For GPU-enabled version, you will also need: From a92a43e282edd95fb45dcf481eb919dbe7afbaf7 Mon Sep 17 00:00:00 2001 From: yukun Date: Mon, 6 Jan 2020 18:35:10 +0800 Subject: [PATCH 03/14] Add push mode for prometheus monitor (#905) * Add push mode for prometheus monitor * format code * fix for comments * fix test_MetricBase bug * Change ip to address in config --- CHANGELOG.md | 1 + core/conf/demo/server_config.yaml | 11 ++++--- core/conf/server_cpu_config.template | 11 ++++--- core/conf/server_gpu_config.template | 11 ++++--- core/src/CMakeLists.txt | 1 + core/src/db/DBImpl.cpp | 1 + core/src/metrics/MetricBase.h | 9 +++-- .../metrics/prometheus/PrometheusMetrics.cpp | 33 ++++++++++++------- .../metrics/prometheus/PrometheusMetrics.h | 25 ++++++++++---- core/src/server/Config.cpp | 26 +++++++++++++++ core/src/server/Config.h | 10 +++++- core/src/server/grpc_impl/GrpcServer.cpp | 1 + core/unittest/metrics/test_metricbase.cpp | 1 + core/unittest/metrics/test_prometheus.cpp | 1 + core/unittest/server/test_config.cpp | 12 +++++-- 15 files changed, 118 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6af48271b..d7a7078dfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the issue from GitHub - \#766 - If partition tag is similar, wrong partition is searched - \#771 - Add server build commit info interface - \#759 - Put C++ sdk out of milvus/core +- \#813 - Add push mode for prometheus monitor - \#815 - Support MinIO storage - \#910 - Change Milvus c++ standard to c++17 diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index e26ac8413b..fa839a1c49 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -20,7 +20,7 @@ version: 0.1 #----------------------+------------------------------------------------------------+------------+-----------------+ # Server Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | String | 0.0.0.0 | +# address | IP address that Milvus server monitors. | Ip | 0.0.0.0 | #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -95,13 +95,16 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # collector | Connected monitoring system to collect metrics. | String | Prometheus | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Port to visit Prometheus, port range (1024, 65535) | Integer | 8080 | +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false collector: prometheus prometheus_config: - port: 8080 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 0f56436774..60d31a1e54 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -20,7 +20,7 @@ version: 0.1 #----------------------+------------------------------------------------------------+------------+-----------------+ # Server Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | String | 0.0.0.0 | +# address | IP address that Milvus server monitors. | Ip | 0.0.0.0 | #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -95,13 +95,16 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # collector | Connected monitoring system to collect metrics. | String | Prometheus | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Port to visit Prometheus, port range (1024, 65535) | Integer | 8080 | +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false collector: prometheus prometheus_config: - port: 8080 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index e622310a97..abc4b4a099 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -20,7 +20,7 @@ version: 0.1 #----------------------+------------------------------------------------------------+------------+-----------------+ # Server Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | String | 0.0.0.0 | +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | String | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -95,13 +95,16 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # collector | Connected monitoring system to collect metrics. | String | Prometheus | #----------------------+------------------------------------------------------------+------------+-----------------+ -# port | Port to visit Prometheus, port range (1024, 65535) | Integer | 8080 | +# address | Pushgateway address | IP | 127.0.0.1 + +#----------------------+------------------------------------------------------------+------------+-----------------+ +# port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false collector: prometheus prometheus_config: - port: 8080 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index 7b3799550e..1199479f88 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -137,6 +137,7 @@ set(prometheus_lib prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core + curl ) set(boost_lib diff --git a/core/src/db/DBImpl.cpp b/core/src/db/DBImpl.cpp index 1e51368fde..ceac415eda 100644 --- a/core/src/db/DBImpl.cpp +++ b/core/src/db/DBImpl.cpp @@ -631,6 +631,7 @@ DBImpl::StartMetricTask() { server::Metrics::GetInstance().CPUCoreUsagePercentSet(); server::Metrics::GetInstance().GPUTemperature(); server::Metrics::GetInstance().CPUTemperature(); + server::Metrics::GetInstance().PushToGateway(); // ENGINE_LOG_TRACE << "Metric task finished"; } diff --git a/core/src/metrics/MetricBase.h b/core/src/metrics/MetricBase.h index eeca45e789..1da4f2726a 100644 --- a/core/src/metrics/MetricBase.h +++ b/core/src/metrics/MetricBase.h @@ -18,7 +18,7 @@ #pragma once #include "SystemInfo.h" -#include "utils/Error.h" +#include "utils/Status.h" #include @@ -32,8 +32,9 @@ class MetricsBase { return instance; } - virtual ErrorCode + virtual Status Init() { + return Status::OK(); } virtual void @@ -203,6 +204,10 @@ class MetricsBase { virtual void CPUTemperature() { } + + virtual void + PushToGateway() { + } }; } // namespace server diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index 19b2683280..c27cf1feb8 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -27,39 +27,48 @@ namespace milvus { namespace server { -ErrorCode +Status PrometheusMetrics::Init() { try { Config& config = Config::GetInstance(); Status s = config.GetMetricConfigEnableMonitor(startup_); if (!s.ok()) { - return s.code(); + return s; } if (!startup_) { - return SERVER_SUCCESS; + return Status::OK(); } // Following should be read from config file. - std::string bind_address; - s = config.GetMetricConfigPrometheusPort(bind_address); + std::string push_port, push_address; + s = config.GetMetricConfigPrometheusPort(push_port); if (!s.ok()) { - return s.code(); + return s; + } + s = config.GetMetricConfigPrometheusAddress(push_address); + if (!s.ok()) { + return s; } const std::string uri = std::string("/metrics"); const std::size_t num_threads = 2; - // Init Exposer - exposer_ptr_ = std::make_shared(bind_address, uri, num_threads); + auto labels = prometheus::Gateway::GetInstanceLabel("pushgateway"); - // Exposer Registry - exposer_ptr_->RegisterCollectable(registry_); + // Init pushgateway + gateway_ = std::make_shared(push_address, push_port, "milvus_metrics", labels); + + // Init Exposer + // exposer_ptr_ = std::make_shared(bind_address, uri, num_threads); + + // Pushgateway Registry + gateway_->RegisterCollectable(registry_); } catch (std::exception& ex) { SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what()); - return SERVER_UNEXPECTED_ERROR; + return Status(SERVER_UNEXPECTED_ERROR, ex.what()); } - return SERVER_SUCCESS; + return Status::OK(); } void diff --git a/core/src/metrics/prometheus/PrometheusMetrics.h b/core/src/metrics/prometheus/PrometheusMetrics.h index 5a452ca02c..f2b56dc362 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.h +++ b/core/src/metrics/prometheus/PrometheusMetrics.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include #include @@ -25,7 +26,8 @@ #include #include "metrics/MetricBase.h" -#include "utils/Error.h" +#include "utils/Log.h" +#include "utils/Status.h" #define METRICS_NOW_TIME std::chrono::system_clock::now() //#define server::Metrics::GetInstance() server::GetInstance() @@ -42,11 +44,11 @@ class PrometheusMetrics : public MetricsBase { return instance; } - ErrorCode - Init(); + Status + Init() override; private: - std::shared_ptr exposer_ptr_; + std::shared_ptr gateway_; std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; @@ -293,9 +295,18 @@ class PrometheusMetrics : public MetricsBase { void CPUTemperature() override; - std::shared_ptr& - exposer_ptr() { - return exposer_ptr_; + void + PushToGateway() override { + if (startup_) { + if (gateway_->Push() != 200) { + ENGINE_LOG_WARNING << "Metrics pushgateway failed"; + } + } + } + + std::shared_ptr& + gateway() { + return gateway_; } // prometheus::Exposer& exposer() { return exposer_;} diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index cfc3959fb4..f5e9a42a82 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -134,6 +134,9 @@ Config::ValidateConfig() { std::string metric_collector; CONFIG_CHECK(GetMetricConfigCollector(metric_collector)); + std::string metric_prometheus_address; + CONFIG_CHECK(GetMetricConfigPrometheusAddress(metric_prometheus_address)); + std::string metric_prometheus_port; CONFIG_CHECK(GetMetricConfigPrometheusPort(metric_prometheus_port)); @@ -214,6 +217,7 @@ Config::ResetDefaultConfig() { /* metric config */ CONFIG_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT)); CONFIG_CHECK(SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT)); + CONFIG_CHECK(SetMetricConfigPrometheusAddress(CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT)); CONFIG_CHECK(SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT)); /* cache config */ @@ -556,6 +560,16 @@ Config::CheckMetricConfigCollector(const std::string& value) { return Status::OK(); } +Status +Config::CheckMetricConfigPrometheusAddress(const std::string& value) { + if (!ValidationUtil::ValidateIpAddress(value).ok()) { + std::string msg = + "Invalid metric ip: " + value + ". Possible reason: metric_config.prometheus_config.ip is invalid."; + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_ip: " + value); + } + return Status::OK(); +} + Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -999,6 +1013,12 @@ Config::GetMetricConfigCollector(std::string& value) { return Status::OK(); } +Status +Config::GetMetricConfigPrometheusAddress(std::string& value) { + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_ADDRESS, CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT); + return Status::OK(); +} + Status Config::GetMetricConfigPrometheusPort(std::string& value) { value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); @@ -1272,6 +1292,12 @@ Config::SetMetricConfigCollector(const std::string& value) { return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } +Status +Config::SetMetricConfigPrometheusAddress(const std::string& value) { + CONFIG_CHECK(CheckMetricConfigPrometheusAddress(value)); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_ADDRESS, value); +} + Status Config::SetMetricConfigPrometheusPort(const std::string& value) { CONFIG_CHECK(CheckMetricConfigPrometheusPort(value)); diff --git a/core/src/server/Config.h b/core/src/server/Config.h index ebc121d4e3..e37bc82647 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -98,8 +98,10 @@ static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; +static const char* CONFIG_METRIC_PROMETHEUS_ADDRESS = "address"; +static const char* CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT = "127.0.0.1"; static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "9091"; /* engine config */ static const char* CONFIG_ENGINE = "engine_config"; @@ -212,6 +214,8 @@ class Config { Status CheckMetricConfigCollector(const std::string& value); Status + CheckMetricConfigPrometheusAddress(const std::string& value); + Status CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ @@ -300,6 +304,8 @@ class Config { Status GetMetricConfigCollector(std::string& value); Status + GetMetricConfigPrometheusAddress(std::string& value); + Status GetMetricConfigPrometheusPort(std::string& value); /* cache config */ @@ -382,6 +388,8 @@ class Config { Status SetMetricConfigCollector(const std::string& value); Status + SetMetricConfigPrometheusAddress(const std::string& value); + Status SetMetricConfigPrometheusPort(const std::string& value); /* cache config */ diff --git a/core/src/server/grpc_impl/GrpcServer.cpp b/core/src/server/grpc_impl/GrpcServer.cpp index 088eb71be0..52cc48b95e 100644 --- a/core/src/server/grpc_impl/GrpcServer.cpp +++ b/core/src/server/grpc_impl/GrpcServer.cpp @@ -55,6 +55,7 @@ class NoReusePortOption : public ::grpc::ServerBuilderOption { void UpdateArguments(::grpc::ChannelArguments* args) override { args->SetInt(GRPC_ARG_ALLOW_REUSEPORT, 0); + args->SetInt(GRPC_ARG_MAX_CONCURRENT_STREAMS, 20); } void diff --git a/core/unittest/metrics/test_metricbase.cpp b/core/unittest/metrics/test_metricbase.cpp index 73c9954e72..f4fd384310 100644 --- a/core/unittest/metrics/test_metricbase.cpp +++ b/core/unittest/metrics/test_metricbase.cpp @@ -62,5 +62,6 @@ TEST(MetricbaseTest, METRICBASE_TEST) { instance.ConnectionGaugeIncrement(); instance.ConnectionGaugeDecrement(); instance.KeepingAliveCounterIncrement(); + instance.PushToGateway(); instance.OctetsSet(); } diff --git a/core/unittest/metrics/test_prometheus.cpp b/core/unittest/metrics/test_prometheus.cpp index 6e339b73b4..d4da5aaa64 100644 --- a/core/unittest/metrics/test_prometheus.cpp +++ b/core/unittest/metrics/test_prometheus.cpp @@ -67,6 +67,7 @@ TEST(PrometheusTest, PROMETHEUS_TEST) { instance.ConnectionGaugeIncrement(); instance.ConnectionGaugeDecrement(); instance.KeepingAliveCounterIncrement(); + instance.PushToGateway(); instance.OctetsSet(); instance.CPUCoreUsagePercentSet(); diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index c669ffa7e1..69259484ac 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -218,6 +218,10 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetMetricConfigCollector(str_val).ok()); ASSERT_TRUE(str_val == metric_collector); + std::string metric_prometheus_address = "127.0.0.1"; + ASSERT_TRUE(config.GetMetricConfigPrometheusAddress(str_val).ok()); + ASSERT_TRUE(str_val == metric_prometheus_address); + std::string metric_prometheus_port = "2222"; ASSERT_TRUE(config.SetMetricConfigPrometheusPort(metric_prometheus_port).ok()); ASSERT_TRUE(config.GetMetricConfigPrometheusPort(str_val).ok()); @@ -298,12 +302,14 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { #endif } -std::string gen_get_command(const std::string& parent_node, const std::string& child_node) { +std::string +gen_get_command(const std::string& parent_node, const std::string& child_node) { std::string cmd = "get_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node; return cmd; } -std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) { +std::string +gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) { std::string cmd = "set_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value; return cmd; } @@ -519,6 +525,8 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(config.SetMetricConfigCollector("zilliz").ok()); + ASSERT_FALSE(config.SetMetricConfigPrometheusAddress("127.0.0").ok()); + ASSERT_FALSE(config.SetMetricConfigPrometheusPort("0xff").ok()); /* cache config */ From 69d42a3d7091e4e59a081c9f706b5ba6fbfade69 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Tue, 7 Jan 2020 16:14:38 +0800 Subject: [PATCH 04/14] remove collector from config (#939) * #931 remove collector from config * #931 fix test_metrics build error --- CHANGELOG.md | 1 + core/conf/demo/server_config.yaml | 14 ++- core/conf/server_cpu_config.template | 14 ++- core/conf/server_gpu_config.template | 12 +-- core/src/metrics/Metrics.cpp | 11 +-- .../metrics/prometheus/PrometheusMetrics.cpp | 17 +--- core/src/server/Config.cpp | 86 +++++++------------ core/src/server/Config.h | 29 +++---- core/unittest/db/utils.cpp | 9 +- core/unittest/metrics/test_metrics.cpp | 8 -- core/unittest/server/test_config.cpp | 28 +++--- core/unittest/server/utils.cpp | 7 +- core/unittest/wrapper/utils.cpp | 9 +- 13 files changed, 81 insertions(+), 164 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7a7078dfd..711614e600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Please mark all change in change log and use the issue from GitHub - \#848 - Add ready-to-use config files to the Milvus repo for enhanced user experince - \#860 - Remove redundant checks in CacheMgr's constructor - \#908 - Move "primary_path" and "secondary_path" to storage config +- \#931 - Remove "collector" from config ## Task diff --git a/core/conf/demo/server_config.yaml b/core/conf/demo/server_config.yaml index fa839a1c49..40cc7855e2 100644 --- a/core/conf/demo/server_config.yaml +++ b/core/conf/demo/server_config.yaml @@ -20,7 +20,7 @@ version: 0.1 #----------------------+------------------------------------------------------------+------------+-----------------+ # Server Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | Ip | 0.0.0.0 | +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -93,23 +93,19 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # enable_monitor | Enable monitoring function or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# collector | Connected monitoring system to collect metrics. | String | Prometheus | -#----------------------+------------------------------------------------------------+------------+-----------------+ # address | Pushgateway address | IP | 127.0.0.1 + #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false - collector: prometheus - prometheus_config: - address: 127.0.0.1 - port: 9091 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integet | 4 (GB) | +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 60d31a1e54..7f375b1126 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -20,7 +20,7 @@ version: 0.1 #----------------------+------------------------------------------------------------+------------+-----------------+ # Server Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# address | IP address that Milvus server monitors. | Ip | 0.0.0.0 | +# address | IP address that Milvus server monitors. | IP | 0.0.0.0 | #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Port that Milvus server monitors. Port range (1024, 65535) | Integer | 19530 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -93,23 +93,19 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # enable_monitor | Enable monitoring function or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# collector | Connected monitoring system to collect metrics. | String | Prometheus | -#----------------------+------------------------------------------------------------+------------+-----------------+ # address | Pushgateway address | IP | 127.0.0.1 + #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false - collector: prometheus - prometheus_config: - address: 127.0.0.1 - port: 9091 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integet | 4 (GB) | +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index abc4b4a099..03861be291 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -68,7 +68,7 @@ db_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_enable | Enable MinIO storage or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# minio_address | MinIO storage service IP address. | Ip | 127.0.0.1 | +# minio_address | MinIO storage service IP address. | IP | 127.0.0.1 | #----------------------+------------------------------------------------------------+------------+-----------------+ # minio_port | MinIO storage service port. Port range (1024, 65535) | Integer | 9000 | #----------------------+------------------------------------------------------------+------------+-----------------+ @@ -93,23 +93,19 @@ storage_config: #----------------------+------------------------------------------------------------+------------+-----------------+ # enable_monitor | Enable monitoring function or not. | Boolean | false | #----------------------+------------------------------------------------------------+------------+-----------------+ -# collector | Connected monitoring system to collect metrics. | String | Prometheus | -#----------------------+------------------------------------------------------------+------------+-----------------+ # address | Pushgateway address | IP | 127.0.0.1 + #----------------------+------------------------------------------------------------+------------+-----------------+ # port | Pushgateway port, port range (1024, 65535) | Integer | 9091 | #----------------------+------------------------------------------------------------+------------+-----------------+ metric_config: enable_monitor: false - collector: prometheus - prometheus_config: - address: 127.0.0.1 - port: 9091 + address: 127.0.0.1 + port: 9091 #----------------------+------------------------------------------------------------+------------+-----------------+ # Cache Config | Description | Type | Default | #----------------------+------------------------------------------------------------+------------+-----------------+ -# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integet | 4 (GB) | +# cpu_cache_capacity | The size of CPU memory used for caching data for faster | Integer | 4 (GB) | # | query. The sum of 'cpu_cache_capacity' and | | | # | 'insert_buffer_size' must be less than system memory size. | | | #----------------------+------------------------------------------------------------+------------+-----------------+ diff --git a/core/src/metrics/Metrics.cpp b/core/src/metrics/Metrics.cpp index 5fd3553cdc..29c9007e22 100644 --- a/core/src/metrics/Metrics.cpp +++ b/core/src/metrics/Metrics.cpp @@ -34,17 +34,8 @@ Metrics::GetInstance() { MetricsBase& Metrics::CreateMetricsCollector() { - Config& config = Config::GetInstance(); - std::string collector_type_str; - - config.GetMetricConfigCollector(collector_type_str); - #ifdef MILVUS_WITH_PROMETHEUS - if (collector_type_str == "prometheus") { - return PrometheusMetrics::GetInstance(); - } else { - return MetricsBase::GetInstance(); - } + return PrometheusMetrics::GetInstance(); #else return MetricsBase::GetInstance(); #endif diff --git a/core/src/metrics/prometheus/PrometheusMetrics.cpp b/core/src/metrics/prometheus/PrometheusMetrics.cpp index c27cf1feb8..f2f5fd06ea 100644 --- a/core/src/metrics/prometheus/PrometheusMetrics.cpp +++ b/core/src/metrics/prometheus/PrometheusMetrics.cpp @@ -31,27 +31,18 @@ Status PrometheusMetrics::Init() { try { Config& config = Config::GetInstance(); - Status s = config.GetMetricConfigEnableMonitor(startup_); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(config.GetMetricConfigEnableMonitor(startup_)); if (!startup_) { return Status::OK(); } // Following should be read from config file. std::string push_port, push_address; - s = config.GetMetricConfigPrometheusPort(push_port); - if (!s.ok()) { - return s; - } - s = config.GetMetricConfigPrometheusAddress(push_address); - if (!s.ok()) { - return s; - } + CONFIG_CHECK(config.GetMetricConfigPort(push_port)); + CONFIG_CHECK(config.GetMetricConfigAddress(push_address)); const std::string uri = std::string("/metrics"); - const std::size_t num_threads = 2; + // const std::size_t num_threads = 2; auto labels = prometheus::Gateway::GetInstanceLabel("pushgateway"); diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index f5e9a42a82..96597ad4a4 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -131,14 +131,11 @@ Config::ValidateConfig() { bool metric_enable_monitor; CONFIG_CHECK(GetMetricConfigEnableMonitor(metric_enable_monitor)); - std::string metric_collector; - CONFIG_CHECK(GetMetricConfigCollector(metric_collector)); + std::string metric_address; + CONFIG_CHECK(GetMetricConfigAddress(metric_address)); - std::string metric_prometheus_address; - CONFIG_CHECK(GetMetricConfigPrometheusAddress(metric_prometheus_address)); - - std::string metric_prometheus_port; - CONFIG_CHECK(GetMetricConfigPrometheusPort(metric_prometheus_port)); + std::string metric_port; + CONFIG_CHECK(GetMetricConfigPort(metric_port)); /* cache config */ int64_t cache_cpu_cache_capacity; @@ -216,9 +213,8 @@ Config::ResetDefaultConfig() { /* metric config */ CONFIG_CHECK(SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT)); - CONFIG_CHECK(SetMetricConfigCollector(CONFIG_METRIC_COLLECTOR_DEFAULT)); - CONFIG_CHECK(SetMetricConfigPrometheusAddress(CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT)); - CONFIG_CHECK(SetMetricConfigPrometheusPort(CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT)); + CONFIG_CHECK(SetMetricConfigAddress(CONFIG_METRIC_ADDRESS_DEFAULT)); + CONFIG_CHECK(SetMetricConfigPort(CONFIG_METRIC_PORT_DEFAULT)); /* cache config */ CONFIG_CHECK(SetCacheConfigCpuCacheCapacity(CONFIG_CACHE_CPU_CACHE_CAPACITY_DEFAULT)); @@ -551,36 +547,24 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) { } Status -Config::CheckMetricConfigCollector(const std::string& value) { - if (value != "prometheus") { - std::string msg = - "Invalid metric collector: " + value + ". Possible reason: metric_config.collector is invalid."; - return Status(SERVER_INVALID_ARGUMENT, msg); - } - return Status::OK(); -} - -Status -Config::CheckMetricConfigPrometheusAddress(const std::string& value) { +Config::CheckMetricConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { - std::string msg = - "Invalid metric ip: " + value + ". Possible reason: metric_config.prometheus_config.ip is invalid."; - return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_ip: " + value); + std::string msg = "Invalid metric ip: " + value + ". Possible reason: metric_config.ip is invalid."; + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config ip: " + value); } return Status::OK(); } Status -Config::CheckMetricConfigPrometheusPort(const std::string& value) { +Config::CheckMetricConfigPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { - std::string msg = "Invalid prometheus port: " + value + - ". Possible reason: metric_config.prometheus_config.port is not a number."; + std::string msg = "Invalid metric port: " + value + ". Possible reason: metric_config.port is not a number."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::string msg = "Invalid prometheus port: " + value + - ". Possible reason: metric_config.prometheus_config.port is not in range (1024, 65535)."; + std::string msg = "Invalid metric port: " + value + + ". Possible reason: metric_config.port is not in range (1024, 65535)."; return Status(SERVER_INVALID_ARGUMENT, msg); } } @@ -948,13 +932,13 @@ Config::GetDBConfigPreloadTable(std::string& value) { /* storage config */ Status Config::GetStorageConfigPrimaryPath(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_STORAGE_PRIMARY_PATH, CONFIG_STORAGE_PRIMARY_PATH_DEFAULT); + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_PRIMARY_PATH, CONFIG_STORAGE_PRIMARY_PATH_DEFAULT); return CheckStorageConfigPrimaryPath(value); } Status Config::GetStorageConfigSecondaryPath(std::string& value) { - value = GetConfigStr(CONFIG_DB, CONFIG_STORAGE_SECONDARY_PATH, CONFIG_STORAGE_SECONDARY_PATH_DEFAULT); + value = GetConfigStr(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, CONFIG_STORAGE_SECONDARY_PATH_DEFAULT); return CheckStorageConfigSecondaryPath(value); } @@ -1008,21 +992,15 @@ Config::GetMetricConfigEnableMonitor(bool& value) { } Status -Config::GetMetricConfigCollector(std::string& value) { - value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); +Config::GetMetricConfigAddress(std::string& value) { + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_ADDRESS, CONFIG_METRIC_ADDRESS_DEFAULT); return Status::OK(); } Status -Config::GetMetricConfigPrometheusAddress(std::string& value) { - value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_ADDRESS, CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT); - return Status::OK(); -} - -Status -Config::GetMetricConfigPrometheusPort(std::string& value) { - value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - return CheckMetricConfigPrometheusPort(value); +Config::GetMetricConfigPort(std::string& value) { + value = GetConfigStr(CONFIG_METRIC, CONFIG_METRIC_PORT, CONFIG_METRIC_PORT_DEFAULT); + return CheckMetricConfigPort(value); } /* cache config */ @@ -1234,19 +1212,19 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) { Status Config::SetStorageConfigPrimaryPath(const std::string& value) { CONFIG_CHECK(CheckStorageConfigPrimaryPath(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_STORAGE_PRIMARY_PATH, value); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_PRIMARY_PATH, value); } Status Config::SetStorageConfigSecondaryPath(const std::string& value) { CONFIG_CHECK(CheckStorageConfigSecondaryPath(value)); - return SetConfigValueInMem(CONFIG_DB, CONFIG_STORAGE_SECONDARY_PATH, value); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_SECONDARY_PATH, value); } Status Config::SetStorageConfigMinioEnable(const std::string& value) { CONFIG_CHECK(CheckStorageConfigMinioEnable(value)); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ENABLE_MONITOR, value); + return SetConfigValueInMem(CONFIG_STORAGE, CONFIG_STORAGE_MINIO_ENABLE, value); } Status @@ -1287,21 +1265,15 @@ Config::SetMetricConfigEnableMonitor(const std::string& value) { } Status -Config::SetMetricConfigCollector(const std::string& value) { - CONFIG_CHECK(CheckMetricConfigCollector(value)); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); +Config::SetMetricConfigAddress(const std::string& value) { + CONFIG_CHECK(CheckMetricConfigAddress(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_ADDRESS, value); } Status -Config::SetMetricConfigPrometheusAddress(const std::string& value) { - CONFIG_CHECK(CheckMetricConfigPrometheusAddress(value)); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_ADDRESS, value); -} - -Status -Config::SetMetricConfigPrometheusPort(const std::string& value) { - CONFIG_CHECK(CheckMetricConfigPrometheusPort(value)); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); +Config::SetMetricConfigPort(const std::string& value) { + CONFIG_CHECK(CheckMetricConfigPort(value)); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PORT, value); } /* cache config */ diff --git a/core/src/server/Config.h b/core/src/server/Config.h index e37bc82647..d3e542db17 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -95,13 +95,10 @@ static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC_ENABLE_MONITOR = "enable_monitor"; static const char* CONFIG_METRIC_ENABLE_MONITOR_DEFAULT = "false"; -static const char* CONFIG_METRIC_COLLECTOR = "collector"; -static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; -static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; -static const char* CONFIG_METRIC_PROMETHEUS_ADDRESS = "address"; -static const char* CONFIG_METRIC_PROMETHEUS_ADDRESS_DEFAULT = "127.0.0.1"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; -static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "9091"; +static const char* CONFIG_METRIC_ADDRESS = "address"; +static const char* CONFIG_METRIC_ADDRESS_DEFAULT = "127.0.0.1"; +static const char* CONFIG_METRIC_PORT = "port"; +static const char* CONFIG_METRIC_PORT_DEFAULT = "9091"; /* engine config */ static const char* CONFIG_ENGINE = "engine_config"; @@ -212,11 +209,9 @@ class Config { Status CheckMetricConfigEnableMonitor(const std::string& value); Status - CheckMetricConfigCollector(const std::string& value); + CheckMetricConfigAddress(const std::string& value); Status - CheckMetricConfigPrometheusAddress(const std::string& value); - Status - CheckMetricConfigPrometheusPort(const std::string& value); + CheckMetricConfigPort(const std::string& value); /* cache config */ Status @@ -302,11 +297,9 @@ class Config { Status GetMetricConfigEnableMonitor(bool& value); Status - GetMetricConfigCollector(std::string& value); + GetMetricConfigAddress(std::string& value); Status - GetMetricConfigPrometheusAddress(std::string& value); - Status - GetMetricConfigPrometheusPort(std::string& value); + GetMetricConfigPort(std::string& value); /* cache config */ Status @@ -386,11 +379,9 @@ class Config { Status SetMetricConfigEnableMonitor(const std::string& value); Status - SetMetricConfigCollector(const std::string& value); + SetMetricConfigAddress(const std::string& value); Status - SetMetricConfigPrometheusAddress(const std::string& value); - Status - SetMetricConfigPrometheusPort(const std::string& value); + SetMetricConfigPort(const std::string& value); /* cache config */ Status diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index db0fc87ff0..97c8250c62 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -61,13 +61,12 @@ static const char* CONFIG_STR = "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" - " collector: prometheus # prometheus\n" - " prometheus_config:\n" - " port: 8080 # port prometheus used to fetch metrics\n" + " address: 127.0.0.1\n" + " port: 9091 # port prometheus used to fetch metrics\n" "\n" "cache_config:\n" - " cpu_mem_capacity: 16 # GB, CPU memory used for cache\n" - " cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" + " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" + " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" " cache_insert_data: false # whether load inserted data into cache\n" "\n" "engine_config:\n" diff --git a/core/unittest/metrics/test_metrics.cpp b/core/unittest/metrics/test_metrics.cpp index 10410a648d..204117f6e5 100644 --- a/core/unittest/metrics/test_metrics.cpp +++ b/core/unittest/metrics/test_metrics.cpp @@ -30,17 +30,9 @@ #include "db/meta/SqliteMetaImpl.h" TEST_F(MetricTest, METRIC_TEST) { - milvus::server::Config::GetInstance().SetMetricConfigCollector("zabbix"); - milvus::server::Metrics::GetInstance(); - milvus::server::Config::GetInstance().SetMetricConfigCollector("prometheus"); - milvus::server::Metrics::GetInstance(); - milvus::server::SystemInfo::GetInstance().Init(); -// server::Metrics::GetInstance().Init(); -// server::Metrics::GetInstance().exposer_ptr()->RegisterCollectable(server::Metrics::GetInstance().registry_ptr()); milvus::server::Metrics::GetInstance().Init(); -// server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr()); milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL * 1024 * 1024 * 1024); std::cout << milvus::cache::CpuCacheMgr::GetInstance()->CacheCapacity() << std::endl; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 69259484ac..6ba839e0b2 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -177,7 +177,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetStorageConfigSecondaryPath(str_val).ok()); ASSERT_TRUE(str_val == storage_secondary_path); - bool storage_minio_enable = false; + bool storage_minio_enable = true; ASSERT_TRUE(config.SetStorageConfigMinioEnable(std::to_string(storage_minio_enable)).ok()); ASSERT_TRUE(config.GetStorageConfigMinioEnable(bool_val).ok()); ASSERT_TRUE(bool_val == storage_minio_enable); @@ -213,19 +213,15 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(config.GetMetricConfigEnableMonitor(bool_val).ok()); ASSERT_TRUE(bool_val == metric_enable_monitor); - std::string metric_collector = "prometheus"; - ASSERT_TRUE(config.SetMetricConfigCollector(metric_collector).ok()); - ASSERT_TRUE(config.GetMetricConfigCollector(str_val).ok()); - ASSERT_TRUE(str_val == metric_collector); + std::string metric_address = "192.168.0.2"; + ASSERT_TRUE(config.SetMetricConfigAddress(metric_address).ok()); + ASSERT_TRUE(config.GetMetricConfigAddress(str_val).ok()); + ASSERT_TRUE(str_val == metric_address); - std::string metric_prometheus_address = "127.0.0.1"; - ASSERT_TRUE(config.GetMetricConfigPrometheusAddress(str_val).ok()); - ASSERT_TRUE(str_val == metric_prometheus_address); - - std::string metric_prometheus_port = "2222"; - ASSERT_TRUE(config.SetMetricConfigPrometheusPort(metric_prometheus_port).ok()); - ASSERT_TRUE(config.GetMetricConfigPrometheusPort(str_val).ok()); - ASSERT_TRUE(str_val == metric_prometheus_port); + std::string metric_port = "2222"; + ASSERT_TRUE(config.SetMetricConfigPort(metric_port).ok()); + ASSERT_TRUE(config.GetMetricConfigPort(str_val).ok()); + ASSERT_TRUE(str_val == metric_port); /* cache config */ int64_t cache_cpu_cache_capacity = 1; @@ -523,11 +519,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { /* metric config */ ASSERT_FALSE(config.SetMetricConfigEnableMonitor("Y").ok()); - ASSERT_FALSE(config.SetMetricConfigCollector("zilliz").ok()); + ASSERT_FALSE(config.SetMetricConfigAddress("127.0.0").ok()); - ASSERT_FALSE(config.SetMetricConfigPrometheusAddress("127.0.0").ok()); - - ASSERT_FALSE(config.SetMetricConfigPrometheusPort("0xff").ok()); + ASSERT_FALSE(config.SetMetricConfigPort("0xff").ok()); /* cache config */ ASSERT_FALSE(config.SetCacheConfigCpuCacheCapacity("a").ok()); diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 301397e047..a1d8c4d340 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -48,12 +48,11 @@ static const char* VALID_CONFIG_STR = "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" - " collector: prometheus # prometheus\n" - " prometheus_config:\n" - " port: 8080 # port prometheus uses to fetch metrics\n" + " address: 127.0.0.1\n" + " port: 8080 # port prometheus uses to fetch metrics\n" "\n" "cache_config:\n" - " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" + " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" " cpu_cache_threshold: 0.85 \n" " cache_insert_data: false # whether to load inserted data into cache\n" "\n" diff --git a/core/unittest/wrapper/utils.cpp b/core/unittest/wrapper/utils.cpp index f4329a4a5f..96cd93511a 100644 --- a/core/unittest/wrapper/utils.cpp +++ b/core/unittest/wrapper/utils.cpp @@ -45,13 +45,12 @@ static const char* CONFIG_STR = "\n" "metric_config:\n" " enable_monitor: false # enable monitoring or not\n" - " collector: prometheus # prometheus\n" - " prometheus_config:\n" - " port: 8080 # port prometheus used to fetch metrics\n" + " address: 127.0.0.1\n" + " port: 8080 # port prometheus used to fetch metrics\n" "\n" "cache_config:\n" - " cpu_mem_capacity: 16 # GB, CPU memory used for cache\n" - " cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" + " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" + " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" " cache_insert_data: false # whether load inserted data into cache\n" "\n" "engine_config:\n" From 7cad4941016b01640618e264a791d1811c8f06a2 Mon Sep 17 00:00:00 2001 From: del-zhenwu <56623710+del-zhenwu@users.noreply.github.com> Date: Tue, 7 Jan 2020 16:20:37 +0800 Subject: [PATCH 05/14] Enhance testing framework based on kubernetes (#935) * Update framework * remove files * Remove files * Remove ann-acc cases && Update java-sdk cases --- tests/milvus-java-test/pom.xml | 4 +- .../src/main/java/com/MainClass.java | 2 +- .../src/main/java/com/Partition.java | 205 ++++++++ .../src/main/java/com/TestAddVectors.java | 65 ++- .../src/main/java/com/TestConnect.java | 19 +- .../src/main/java/com/TestIndex.java | 2 +- .../src/main/java/com/TestMix.java | 25 + .../src/main/java/com/TestPS.java | 115 +++++ .../src/main/java/com/TestSearchVectors.java | 139 ++++- tests/milvus_ann_acc/.gitignore | 2 - tests/milvus_ann_acc/README.md | 21 - .../ci/jenkinsfile/acc_test.groovy | 16 - .../ci/jenkinsfile/cleanup.groovy | 13 - .../jenkinsfile/deploy_default_server.groovy | 22 - tests/milvus_ann_acc/client.py | 146 ------ tests/milvus_ann_acc/config.yaml | 17 - tests/milvus_ann_acc/main.py | 57 --- tests/milvus_ann_acc/requirements.txt | 9 - tests/milvus_ann_acc/runner.py | 162 ------ tests/milvus_ann_acc/suite.yaml | 29 -- tests/milvus_ann_acc/suite.yaml.bak | 11 - tests/milvus_ann_acc/suite_cpu.yaml | 19 - tests/milvus_ann_acc/suite_czr.yaml | 20 - tests/milvus_ann_acc/suite_debug.yaml | 19 - tests/milvus_ann_acc/suite_gpu.yaml | 19 - tests/milvus_ann_acc/test.py | 33 -- tests/milvus_benchmark/README.md | 49 +- tests/milvus_benchmark/assets/Parameters.png | Bin 0 -> 50835 bytes .../gpu_search_performance_random50m-yaml.png | Bin 0 -> 66119 bytes ...milvus-nightly-performance-new-jenkins.png | Bin 0 -> 45034 bytes .../ci/function/file_transfer.groovy | 0 .../ci/jenkinsfile/cleanup.groovy | 13 + .../ci/jenkinsfile/deploy_test.groovy | 20 + .../ci/jenkinsfile/notify.groovy | 0 .../ci/main_jenkinsfile | 59 +-- .../pod_containers/milvus-testframework.yaml | 0 tests/milvus_benchmark/client.py | 107 +++- tests/milvus_benchmark/demo.py | 51 -- tests/milvus_benchmark/docker_runner.py | 91 +++- tests/milvus_benchmark/k8s_runner.py | 473 ++++++++++++++++++ tests/milvus_benchmark/local_runner.py | 10 +- tests/milvus_benchmark/main.py | 137 +++-- tests/milvus_benchmark/parser.py | 26 +- tests/milvus_benchmark/requirements.txt | 3 +- tests/milvus_benchmark/runner.py | 106 ++-- .../scripts/default_config.json | 86 ++++ tests/milvus_benchmark/scripts/scheduler.py | 28 ++ tests/milvus_benchmark/suites.yaml | 38 -- .../milvus_benchmark/suites/cpu_accuracy.yaml | 119 +++++ .../suites/cpu_accuracy_ann.yaml | 68 +++ .../suites/cpu_build_performance.yaml | 36 ++ .../suites/cpu_search_performance.yaml | 169 +++++++ .../suites/cpu_search_stability.yaml | 20 + .../suites/cpu_stability_sift50m.yaml | 27 + .../milvus_benchmark/suites/gpu_accuracy.yaml | 157 ++++++ .../suites/gpu_accuracy_ann.yaml | 68 +++ .../suites/gpu_accuracy_ann_debug.yaml | 47 ++ .../suites/gpu_accuracy_sift1b.yaml | 59 +++ .../suites/gpu_accuracy_sift1m.yaml | 40 ++ .../suites/gpu_accuracy_sift50m.yaml | 80 +++ .../suites/gpu_build_performance.yaml | 36 ++ .../gpu_build_performance_hamming50m.yaml | 36 ++ .../gpu_build_performance_jaccard50m.yaml | 36 ++ .../suites/gpu_search_performance.yaml | 247 +++++++++ .../gpu_search_performance_hamming50m.yaml | 22 + .../gpu_search_performance_jaccard50m.yaml | 22 + .../gpu_search_performance_random50m.yaml | 82 +++ .../suites/gpu_search_performance_sift1b.yaml | 62 +++ .../suites/gpu_search_performance_sift1m.yaml | 42 ++ .../gpu_search_performance_sift50m.yaml | 146 ++++++ .../suites/gpu_search_stability.yaml | 23 + .../suites/gpu_stability_sift1m.yaml | 26 + .../suites/gpu_stability_sift50m.yaml | 27 + .../suites/insert_performance.yaml | 19 + .../suites/insert_performance_deep1b.yaml | 87 ++++ .../suites/insert_performance_hamming50m.yaml | 36 ++ .../suites/insert_performance_jaccard50m.yaml | 36 ++ .../suites/insert_performance_sift50m.yaml | 87 ++++ .../suites/old_performance_sift50m.yaml | 80 --- .../milvus_benchmark/suites/search_debug.yaml | 22 + tests/milvus_benchmark/suites_accuracy.yaml | 121 ----- .../milvus_benchmark/suites_performance.yaml | 258 ---------- tests/milvus_benchmark/suites_stability.yaml | 17 - tests/milvus_benchmark/suites_yzb.yaml | 171 ------- tests/milvus_benchmark/utils.py | 387 +++++++++----- 85 files changed, 3828 insertions(+), 1678 deletions(-) create mode 100644 tests/milvus-java-test/src/main/java/com/Partition.java create mode 100644 tests/milvus-java-test/src/main/java/com/TestPS.java delete mode 100644 tests/milvus_ann_acc/.gitignore delete mode 100644 tests/milvus_ann_acc/README.md delete mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy delete mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy delete mode 100644 tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy delete mode 100644 tests/milvus_ann_acc/client.py delete mode 100644 tests/milvus_ann_acc/config.yaml delete mode 100644 tests/milvus_ann_acc/main.py delete mode 100644 tests/milvus_ann_acc/requirements.txt delete mode 100644 tests/milvus_ann_acc/runner.py delete mode 100644 tests/milvus_ann_acc/suite.yaml delete mode 100644 tests/milvus_ann_acc/suite.yaml.bak delete mode 100644 tests/milvus_ann_acc/suite_cpu.yaml delete mode 100644 tests/milvus_ann_acc/suite_czr.yaml delete mode 100644 tests/milvus_ann_acc/suite_debug.yaml delete mode 100644 tests/milvus_ann_acc/suite_gpu.yaml delete mode 100644 tests/milvus_ann_acc/test.py create mode 100644 tests/milvus_benchmark/assets/Parameters.png create mode 100644 tests/milvus_benchmark/assets/gpu_search_performance_random50m-yaml.png create mode 100644 tests/milvus_benchmark/assets/milvus-nightly-performance-new-jenkins.png rename tests/{milvus_ann_acc => milvus_benchmark}/ci/function/file_transfer.groovy (100%) create mode 100644 tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy create mode 100644 tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy rename tests/{milvus_ann_acc => milvus_benchmark}/ci/jenkinsfile/notify.groovy (100%) rename tests/{milvus_ann_acc => milvus_benchmark}/ci/main_jenkinsfile (66%) rename tests/{milvus_ann_acc => milvus_benchmark}/ci/pod_containers/milvus-testframework.yaml (100%) delete mode 100644 tests/milvus_benchmark/demo.py create mode 100644 tests/milvus_benchmark/k8s_runner.py create mode 100644 tests/milvus_benchmark/scripts/default_config.json create mode 100644 tests/milvus_benchmark/scripts/scheduler.py delete mode 100644 tests/milvus_benchmark/suites.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_build_performance.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_performance.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_search_stability.yaml create mode 100644 tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_performance.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_search_stability.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml create mode 100644 tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance_deep1b.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml create mode 100644 tests/milvus_benchmark/suites/insert_performance_sift50m.yaml delete mode 100644 tests/milvus_benchmark/suites/old_performance_sift50m.yaml create mode 100644 tests/milvus_benchmark/suites/search_debug.yaml delete mode 100644 tests/milvus_benchmark/suites_accuracy.yaml delete mode 100644 tests/milvus_benchmark/suites_performance.yaml delete mode 100644 tests/milvus_benchmark/suites_stability.yaml delete mode 100644 tests/milvus_benchmark/suites_yzb.yaml diff --git a/tests/milvus-java-test/pom.xml b/tests/milvus-java-test/pom.xml index bff6f1de61..3d5e7511f3 100644 --- a/tests/milvus-java-test/pom.xml +++ b/tests/milvus-java-test/pom.xml @@ -99,7 +99,7 @@ io.milvus milvus-sdk-java - 0.3.0 + 0.4.1-SNAPSHOT @@ -134,4 +134,4 @@ - + \ No newline at end of file diff --git a/tests/milvus-java-test/src/main/java/com/MainClass.java b/tests/milvus-java-test/src/main/java/com/MainClass.java index 8928843f01..1eaf082d58 100644 --- a/tests/milvus-java-test/src/main/java/com/MainClass.java +++ b/tests/milvus-java-test/src/main/java/com/MainClass.java @@ -15,7 +15,7 @@ import java.util.List; public class MainClass { private static String host = "127.0.0.1"; - private static String port = "19530"; + private static String port = "19532"; private int index_file_size = 50; public int dimension = 128; diff --git a/tests/milvus-java-test/src/main/java/com/Partition.java b/tests/milvus-java-test/src/main/java/com/Partition.java new file mode 100644 index 0000000000..89a12663ea --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/Partition.java @@ -0,0 +1,205 @@ +package com; + +import io.milvus.client.MilvusClient; +import io.milvus.client.Response; +import io.milvus.client.ShowPartitionsResponse; +import org.apache.commons.lang3.RandomStringUtils; +import org.testng.Assert; +import org.testng.annotations.Test; + +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Random; + +public class Partition { + int dimension = 128; + + public List> gen_vectors(Integer nb) { + List> xb = new LinkedList<>(); + Random random = new Random(); + for (int i = 0; i < nb; ++i) { + LinkedList vector = new LinkedList<>(); + for (int j = 0; j < dimension; j++) { + vector.add(random.nextFloat()); + } + xb.add(vector); + } + return xb; + } + + // ----------------------------- create partition cases in --------------------------------- + + // create partition + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_create_partition(MilvusClient client, String tableName) { + String partitionName = RandomStringUtils.randomAlphabetic(10); + String tag = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + Response createpResponse = client.createPartition(partition); + assert (createpResponse.ok()); + // show partitions + List partitions = client.showPartitions(tableName).getPartitionList(); + System.out.println(partitions); + List partitionNames = new ArrayList<>(); + for (int i=0; i partitionNames = new ArrayList<>(); + for (int i=0; i partitionNames = response.getPartitionNameList(); +// for (int i=0; i tagNames = response.getPartitionTagList(); + Assert.assertTrue(tagNames.contains(tag)); + Assert.assertTrue(tagNames.contains(tagNew)); + } + +} diff --git a/tests/milvus-java-test/src/main/java/com/TestAddVectors.java b/tests/milvus-java-test/src/main/java/com/TestAddVectors.java index 215f526179..dfb1866bb3 100644 --- a/tests/milvus-java-test/src/main/java/com/TestAddVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestAddVectors.java @@ -1,8 +1,7 @@ package com; -import io.milvus.client.InsertParam; -import io.milvus.client.InsertResponse; -import io.milvus.client.MilvusClient; +import io.milvus.client.*; +import org.apache.commons.lang3.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.Test; @@ -12,6 +11,7 @@ import java.util.stream.Stream; public class TestAddVectors { int dimension = 128; + String tag = "tag"; public List> gen_vectors(Integer nb) { List> xb = new LinkedList<>(); @@ -28,7 +28,7 @@ public class TestAddVectors { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_vectors_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { - int nb = 10000; + int nb = 1000; List> vectors = gen_vectors(nb); String tableNameNew = tableName + "_"; InsertParam insertParam = new InsertParam.Builder(tableNameNew, vectors).build(); @@ -47,7 +47,7 @@ public class TestAddVectors { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_vectors(MilvusClient client, String tableName) throws InterruptedException { - int nb = 10000; + int nb = 1000; List> vectors = gen_vectors(nb); InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); InsertResponse res = client.insert(insertParam); @@ -79,7 +79,7 @@ public class TestAddVectors { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_vectors_with_ids(MilvusClient client, String tableName) throws InterruptedException { - int nb = 10000; + int nb = 1000; List> vectors = gen_vectors(nb); // Add vectors with ids List vectorIds; @@ -111,7 +111,7 @@ public class TestAddVectors { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_vectors_with_invalid_dimension(MilvusClient client, String tableName) { - int nb = 10000; + int nb = 1000; List> vectors = gen_vectors(nb); vectors.get(0).add((float) 0); InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); @@ -121,7 +121,7 @@ public class TestAddVectors { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_vectors_with_invalid_vectors(MilvusClient client, String tableName) { - int nb = 10000; + int nb = 1000; List> vectors = gen_vectors(nb); vectors.set(0, new ArrayList<>()); InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); @@ -147,4 +147,53 @@ public class TestAddVectors { Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb * loops); } + // ----------------------------- partition cases in Insert --------------------------------- + // Add vectors into table with given tag + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_add_vectors_partition(MilvusClient client, String tableName) throws InterruptedException { + int nb = 1000; + List> vectors = gen_vectors(nb); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + Response createpResponse = client.createPartition(partition); + assert(createpResponse.ok()); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + Thread.currentThread().sleep(1000); + // Assert table row count + Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + } + + // Add vectors into table, which tag not existed + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_tag_not_existed(MilvusClient client, String tableName) { + int nb = 1000; + String newTag = RandomStringUtils.randomAlphabetic(10); + List> vectors = gen_vectors(nb); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + Response createpResponse = client.createPartition(partition); + assert(createpResponse.ok()); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(newTag).build(); + InsertResponse res = client.insert(insertParam); + assert(!res.getResponse().ok()); + } + + // Create table, add vectors into table + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_A(MilvusClient client, String tableName) throws InterruptedException { + int nb = 1000; + List> vectors = gen_vectors(nb); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + Response createpResponse = client.createPartition(partition); + assert(createpResponse.ok()); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); + InsertResponse res = client.insert(insertParam); + assert(res.getResponse().ok()); + Thread.currentThread().sleep(1000); + // Assert table row count + Assert.assertEquals(client.getTableRowCount(tableName).getTableRowCount(), nb); + } } diff --git a/tests/milvus-java-test/src/main/java/com/TestConnect.java b/tests/milvus-java-test/src/main/java/com/TestConnect.java index 8f6d556f8b..ef89bb97ee 100644 --- a/tests/milvus-java-test/src/main/java/com/TestConnect.java +++ b/tests/milvus-java-test/src/main/java/com/TestConnect.java @@ -22,12 +22,13 @@ public class TestConnect { @Test(dataProvider = "DefaultConnectArgs", dataProviderClass = MainClass.class) public void test_connect_repeat(String host, String port) { MilvusGrpcClient client = new MilvusGrpcClient(); - ConnectParam connectParam = new ConnectParam.Builder() - .withHost(host) - .withPort(port) - .build(); + Response res = null; try { + ConnectParam connectParam = new ConnectParam.Builder() + .withHost(host) + .withPort(port) + .build(); res = client.connect(connectParam); res = client.connect(connectParam); } catch (ConnectFailedException e) { @@ -40,14 +41,14 @@ public class TestConnect { @Test(dataProvider="InvalidConnectArgs") public void test_connect_invalid_connect_args(String ip, String port) { MilvusClient client = new MilvusGrpcClient(); - ConnectParam connectParam = new ConnectParam.Builder() - .withHost(ip) - .withPort(port) - .build(); Response res = null; try { + ConnectParam connectParam = new ConnectParam.Builder() + .withHost(ip) + .withPort(port) + .build(); res = client.connect(connectParam); - } catch (ConnectFailedException e) { + } catch (Exception e) { e.printStackTrace(); } Assert.assertEquals(res, null); diff --git a/tests/milvus-java-test/src/main/java/com/TestIndex.java b/tests/milvus-java-test/src/main/java/com/TestIndex.java index eaf0c8dc10..a87befd97a 100644 --- a/tests/milvus-java-test/src/main/java/com/TestIndex.java +++ b/tests/milvus-java-test/src/main/java/com/TestIndex.java @@ -134,7 +134,7 @@ public class TestIndex { @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_create_index_IVFSQ8H(MilvusClient client, String tableName) throws InterruptedException { - IndexType indexType = IndexType.IVF_SQ8H; + IndexType indexType = IndexType.IVF_SQ8_H; List> vectors = gen_vectors(nb); InsertParam insertParam = new InsertParam.Builder(tableName, vectors).build(); client.insert(insertParam); diff --git a/tests/milvus-java-test/src/main/java/com/TestMix.java b/tests/milvus-java-test/src/main/java/com/TestMix.java index 7c33da7094..35f514d136 100644 --- a/tests/milvus-java-test/src/main/java/com/TestMix.java +++ b/tests/milvus-java-test/src/main/java/com/TestMix.java @@ -120,6 +120,31 @@ public class TestMix { Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); } + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_add_vectors_partition_threads(MilvusClient client, String tableName) throws InterruptedException { + int thread_num = 10; + String tag = RandomStringUtils.randomAlphabetic(10); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + client.createPartition(partition); + List> vectors = gen_vectors(nb,false); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + ForkJoinPool executor = new ForkJoinPool(); + for (int i = 0; i < thread_num; i++) { + executor.execute( + () -> { + InsertResponse res_insert = client.insert(insertParam); + assert (res_insert.getResponse().ok()); + }); + } + executor.awaitQuiescence(100, TimeUnit.SECONDS); + executor.shutdown(); + + Thread.sleep(2000); + GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); + Assert.assertEquals(getTableRowCountResponse.getTableRowCount(), thread_num * nb); + } + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) public void test_add_index_vectors_threads(MilvusClient client, String tableName) throws InterruptedException { int thread_num = 50; diff --git a/tests/milvus-java-test/src/main/java/com/TestPS.java b/tests/milvus-java-test/src/main/java/com/TestPS.java new file mode 100644 index 0000000000..cfce542d1b --- /dev/null +++ b/tests/milvus-java-test/src/main/java/com/TestPS.java @@ -0,0 +1,115 @@ +package com; + +import io.milvus.client.*; +import org.apache.commons.cli.*; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.ForkJoinPool; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class TestPS { + private static int dimension = 128; + private static String host = "192.168.1.101"; + private static String port = "19530"; + + public static void setHost(String host) { + TestPS.host = host; + } + + public static void setPort(String port) { + TestPS.port = port; + } + + public static List normalize(List w2v){ + float squareSum = w2v.stream().map(x -> x * x).reduce((float) 0, Float::sum); + final float norm = (float) Math.sqrt(squareSum); + w2v = w2v.stream().map(x -> x / norm).collect(Collectors.toList()); + return w2v; + } + + public static List> gen_vectors(int nb, boolean norm) { + List> xb = new ArrayList<>(); + Random random = new Random(); + for (int i = 0; i < nb; ++i) { + List vector = new ArrayList<>(); + for (int j = 0; j < dimension; j++) { + vector.add(random.nextFloat()); + } + if (norm == true) { + vector = normalize(vector); + } + xb.add(vector); + } + return xb; + } + + public static void main(String[] args) throws ConnectFailedException { + int nq = 5; + int nb = 1; + int nprobe = 32; + int top_k = 10; + int loops = 100000; +// int index_file_size = 1024; + String tableName = "sift_1b_2048_128_l2"; + + CommandLineParser parser = new DefaultParser(); + Options options = new Options(); + options.addOption("h", "host", true, "milvus-server hostname/ip"); + options.addOption("p", "port", true, "milvus-server port"); + try { + CommandLine cmd = parser.parse(options, args); + String host = cmd.getOptionValue("host"); + if (host != null) { + setHost(host); + } + String port = cmd.getOptionValue("port"); + if (port != null) { + setPort(port); + } + System.out.println("Host: "+host+", Port: "+port); + } + catch(ParseException exp) { + System.err.println("Parsing failed. Reason: " + exp.getMessage() ); + } + + List> vectors = gen_vectors(nb, true); + List> queryVectors = gen_vectors(nq, true); + MilvusClient client = new MilvusGrpcClient(); + ConnectParam connectParam = new ConnectParam.Builder() + .withHost(host) + .withPort(port) + .build(); + client.connect(connectParam); +// String tableName = RandomStringUtils.randomAlphabetic(10); +// TableSchema tableSchema = new TableSchema.Builder(tableName, dimension) +// .withIndexFileSize(index_file_size) +// .withMetricType(MetricType.IP) +// .build(); +// Response res = client.createTable(tableSchema); + List vectorIds; + vectorIds = Stream.iterate(0L, n -> n) + .limit(nb) + .collect(Collectors.toList()); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withVectorIds(vectorIds).build(); + ForkJoinPool executor_search = new ForkJoinPool(); + for (int i = 0; i < loops; i++) { + executor_search.execute( + () -> { + InsertResponse res_insert = client.insert(insertParam); + assert (res_insert.getResponse().ok()); + System.out.println("In insert"); + SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(nprobe).withTopK(top_k).build(); + SearchResponse res_search = client.search(searchParam); + assert (res_search.getResponse().ok()); + }); + } + executor_search.awaitQuiescence(300, TimeUnit.SECONDS); + executor_search.shutdown(); + GetTableRowCountResponse getTableRowCountResponse = client.getTableRowCount(tableName); + System.out.println(getTableRowCountResponse.getTableRowCount()); + } +} diff --git a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java index de69a1c065..3cd4f7de93 100644 --- a/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java +++ b/tests/milvus-java-test/src/main/java/com/TestSearchVectors.java @@ -1,10 +1,12 @@ package com; import io.milvus.client.*; +import org.apache.commons.lang3.RandomStringUtils; import org.testng.Assert; import org.testng.annotations.Test; import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -53,7 +55,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_table_not_existed(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_table_not_existed(MilvusClient client, String tableName) { String tableNameNew = tableName + "_"; int nq = 5; int nb = 100; @@ -65,7 +67,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_IVFLAT(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_index_IVFLAT(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVFLAT; int nq = 5; List> vectors = gen_vectors(nb, false); @@ -84,7 +86,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_ids_IVFLAT(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_ids_IVFLAT(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVFLAT; int nq = 5; List> vectors = gen_vectors(nb, true); @@ -121,7 +123,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_IVFLAT(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_distance_IVFLAT(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVFLAT; int nq = 5; List> vectors = gen_vectors(nb, true); @@ -144,7 +146,120 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_distance_partition(MilvusClient client, String tableName) { + IndexType indexType = IndexType.IVFLAT; + int nq = 5; + String tag = RandomStringUtils.randomAlphabetic(10); + List> vectors = gen_vectors(nb, true); + List> queryVectors = vectors.subList(0,nq); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + client.createPartition(partition); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + client.insert(insertParam); + Index index = new Index.Builder().withIndexType(indexType) + .withNList(n_list) + .build(); + CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); + client.createIndex(createIndexParam); + SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); + List> res_search = client.search(searchParam).getQueryResultsList(); + double distance = res_search.get(0).get(0).getDistance(); + if (tableName.startsWith("L2")) { + Assert.assertEquals(distance, 0.0, epsilon); + }else if (tableName.startsWith("IP")) { + Assert.assertEquals(distance, 1.0, epsilon); + } + } + + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_search_distance_partition_not_exited(MilvusClient client, String tableName) { + IndexType indexType = IndexType.IVFLAT; + int nq = 5; + String tag = RandomStringUtils.randomAlphabetic(10); + List> vectors = gen_vectors(nb, true); + List> queryVectors = vectors.subList(0,nq); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + client.createPartition(partition); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + client.insert(insertParam); + Index index = new Index.Builder().withIndexType(indexType) + .withNList(n_list) + .build(); + CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); + client.createIndex(createIndexParam); + String tagNew = RandomStringUtils.randomAlphabetic(10); + List queryTags = new ArrayList<>(); + queryTags.add(tagNew); + SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); + SearchResponse res_search = client.search(searchParam); + assert (res_search.getResponse().ok()); + Assert.assertEquals(res_search.getQueryResultsList().size(), 0); + } + + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_search_distance_partition_empty(MilvusClient client, String tableName) { + IndexType indexType = IndexType.IVFLAT; + int nq = 5; + String tag = RandomStringUtils.randomAlphabetic(10); + List> vectors = gen_vectors(nb, true); + List> queryVectors = vectors.subList(0,nq); + String partitionName = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + client.createPartition(partition); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + client.insert(insertParam); + Index index = new Index.Builder().withIndexType(indexType) + .withNList(n_list) + .build(); + CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); + client.createIndex(createIndexParam); + String tagNew = ""; + List queryTags = new ArrayList<>(); + queryTags.add(tagNew); + SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); + SearchResponse res_search = client.search(searchParam); + assert (!res_search.getResponse().ok()); + } + + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_search_distance_partition_A(MilvusClient client, String tableName) throws InterruptedException { +// IndexType indexType = IndexType.IVFLAT; + String tag = RandomStringUtils.randomAlphabetic(10); + String tagNew = RandomStringUtils.randomAlphabetic(10); + List> vectors = gen_vectors(nb, true); + List> vectorsNew = gen_vectors(nb, true); + String partitionName = RandomStringUtils.randomAlphabetic(10); + String partitionNameNew = RandomStringUtils.randomAlphabetic(10); + io.milvus.client.Partition partition = new io.milvus.client.Partition.Builder(tableName, partitionName, tag).build(); + io.milvus.client.Partition partitionNew = new io.milvus.client.Partition.Builder(tableName, partitionNameNew, tagNew).build(); + client.createPartition(partition); + client.createPartition(partitionNew); + InsertParam insertParam = new InsertParam.Builder(tableName, vectors).withPartitionTag(tag).build(); + InsertResponse res = client.insert(insertParam); + System.out.println(res.getVectorIds().size()); + InsertParam insertParamNew = new InsertParam.Builder(tableName, vectorsNew).withPartitionTag(tagNew).build(); + InsertResponse resNew = client.insert(insertParamNew); + TimeUnit.SECONDS.sleep(2); + System.out.println(resNew.getVectorIds().size()); + List queryTags = new ArrayList<>(); + queryTags.add(tag); + List> queryVectors; + queryVectors = vectors.subList(0,2); + queryVectors.add(vectorsNew.get(0)); + System.out.println(queryVectors.size()); + SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).withPartitionTags(queryTags).build(); + List> res_search = client.search(searchParam).getResultIdsList(); + System.out.println(res_search.get(0)); + System.out.println(res.getVectorIds()); +// System.out.println(res_search.get(2)); + Assert.assertTrue(res.getVectorIds().containsAll(res_search.get(0))); + Assert.assertTrue(resNew.getVectorIds().contains(res_search.get(2))); + } + + @Test(dataProvider = "Table", dataProviderClass = MainClass.class) + public void test_search_index_IVFSQ8(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVF_SQ8; int nq = 5; List> vectors = gen_vectors(nb, false); @@ -178,7 +293,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_IVFSQ8(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_distance_IVFSQ8(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVF_SQ8; int nq = 5; int nb = 1000; @@ -192,7 +307,7 @@ public class TestSearchVectors { CreateIndexParam createIndexParam = new CreateIndexParam.Builder(tableName).withIndex(index).build(); client.createIndex(createIndexParam); SearchParam searchParam = new SearchParam.Builder(tableName, queryVectors).withNProbe(n_probe).withTopK(top_k).build(); - List> res_search = client.search(searchParam).getResultDistancesList(); + List> res_search = client.search(searchParam).getResultDistancesList(); for (int i = 0; i < nq; i++) { double distance = res_search.get(i).get(0); System.out.println(distance); @@ -205,7 +320,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_FLAT(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_index_FLAT(MilvusClient client, String tableName) { IndexType indexType = IndexType.FLAT; int nq = 5; List> vectors = gen_vectors(nb, false); @@ -274,7 +389,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_distance_FLAT(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_distance_FLAT(MilvusClient client, String tableName) { IndexType indexType = IndexType.FLAT; int nq = 5; List> vectors = gen_vectors(nb, true); @@ -297,7 +412,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_invalid_n_probe(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_invalid_n_probe(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVF_SQ8; int nq = 5; int n_probe_new = 0; @@ -316,7 +431,7 @@ public class TestSearchVectors { } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_invalid_top_k(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_invalid_top_k(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVF_SQ8; int nq = 5; int top_k_new = 0; @@ -353,7 +468,7 @@ public class TestSearchVectors { // } @Test(dataProvider = "Table", dataProviderClass = MainClass.class) - public void test_search_index_range(MilvusClient client, String tableName) throws InterruptedException { + public void test_search_index_range(MilvusClient client, String tableName) { IndexType indexType = IndexType.IVF_SQ8; int nq = 5; List> vectors = gen_vectors(nb, false); diff --git a/tests/milvus_ann_acc/.gitignore b/tests/milvus_ann_acc/.gitignore deleted file mode 100644 index f250cab9fe..0000000000 --- a/tests/milvus_ann_acc/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -__pycache__/ -logs/ diff --git a/tests/milvus_ann_acc/README.md b/tests/milvus_ann_acc/README.md deleted file mode 100644 index f5ab9d8168..0000000000 --- a/tests/milvus_ann_acc/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Requirements - -- python 3.6+ -- pip install -r requirements.txt - -# How to use this Test Project - -This project is used to test search accuracy based on the given datasets (https://github.com/erikbern/ann-benchmarks#data-sets) - -1. start your milvus server -2. update your test configuration in test.py -3. run command - -```shell -python test.py -``` - -# Contribution getting started - -- Follow PEP-8 for naming and black for formatting. - diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy deleted file mode 100644 index c1b2b2ed64..0000000000 --- a/tests/milvus_ann_acc/ci/jenkinsfile/acc_test.groovy +++ /dev/null @@ -1,16 +0,0 @@ -timeout(time: 7200, unit: 'MINUTES') { - try { - dir ("milvu_ann_acc") { - print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" - checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) - print "Install requirements" - sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - // sleep(120000) - sh "python3 main.py --suite=${params.SUITE} --host=acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER}-engine.milvus.svc.cluster.local --port=19530" - } - } catch (exc) { - echo 'Milvus Ann Accuracy Test Failed !' - throw exc - } -} - diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy deleted file mode 100644 index 2e9332fa6e..0000000000 --- a/tests/milvus_ann_acc/ci/jenkinsfile/cleanup.groovy +++ /dev/null @@ -1,13 +0,0 @@ -try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } -} catch (exc) { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (!result) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - } - throw exc -} - diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy b/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy deleted file mode 100644 index 951bb69941..0000000000 --- a/tests/milvus_ann_acc/ci/jenkinsfile/deploy_default_server.groovy +++ /dev/null @@ -1,22 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - try { - dir ("milvus") { - sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' - sh 'helm repo update' - checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) - dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${IMAGE_TAG} --set expose.type=clusterIP --name acc-test-${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/sqlite_${params.IMAGE_TYPE}_values.yaml -f ci/filebeat/values.yaml --namespace milvus --version ${HELM_BRANCH} ." - } - } - // dir ("milvus") { - // checkout([$class: 'GitSCM', branches: [[name: "${env.SERVER_BRANCH}"]], userRemoteConfigs: [[url: "${env.SERVER_URL}", name: 'origin', refspec: "+refs/heads/${env.SERVER_BRANCH}:refs/remotes/origin/${env.SERVER_BRANCH}"]]]) - // dir ("milvus") { - // load "ci/jenkins/step/deploySingle2Dev.groovy" - // } - // } - } catch (exc) { - echo 'Deploy Milvus Server Failed !' - throw exc - } -} - diff --git a/tests/milvus_ann_acc/client.py b/tests/milvus_ann_acc/client.py deleted file mode 100644 index 4ec3cb151a..0000000000 --- a/tests/milvus_ann_acc/client.py +++ /dev/null @@ -1,146 +0,0 @@ -import pdb -import random -import logging -import json -import time, datetime -from multiprocessing import Process -import numpy -import sklearn.preprocessing -from milvus import Milvus, IndexType, MetricType - -logger = logging.getLogger("milvus_acc.client") - -SERVER_HOST_DEFAULT = "127.0.0.1" -SERVER_PORT_DEFAULT = 19530 - - -def time_wrapper(func): - """ - This decorator prints the execution time for the decorated function. - """ - def wrapper(*args, **kwargs): - start = time.time() - result = func(*args, **kwargs) - end = time.time() - logger.info("Milvus {} run in {}s".format(func.__name__, round(end - start, 2))) - return result - return wrapper - - -class MilvusClient(object): - def __init__(self, table_name=None, host=None, port=None): - self._milvus = Milvus() - self._table_name = table_name - try: - if not host: - self._milvus.connect( - host = SERVER_HOST_DEFAULT, - port = SERVER_PORT_DEFAULT) - else: - self._milvus.connect( - host = host, - port = port) - except Exception as e: - raise e - - def __str__(self): - return 'Milvus table %s' % self._table_name - - def check_status(self, status): - if not status.OK(): - logger.error(status.message) - raise Exception("Status not ok") - - def create_table(self, table_name, dimension, index_file_size, metric_type): - if not self._table_name: - self._table_name = table_name - if metric_type == "l2": - metric_type = MetricType.L2 - elif metric_type == "ip": - metric_type = MetricType.IP - else: - logger.error("Not supported metric_type: %s" % metric_type) - self._metric_type = metric_type - create_param = {'table_name': table_name, - 'dimension': dimension, - 'index_file_size': index_file_size, - "metric_type": metric_type} - status = self._milvus.create_table(create_param) - self.check_status(status) - - @time_wrapper - def insert(self, X, ids): - if self._metric_type == MetricType.IP: - logger.info("Set normalize for metric_type: Inner Product") - X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') - X = X.astype(numpy.float32) - status, result = self._milvus.add_vectors(self._table_name, X.tolist(), ids=ids) - self.check_status(status) - return status, result - - @time_wrapper - def create_index(self, index_type, nlist): - if index_type == "flat": - index_type = IndexType.FLAT - elif index_type == "ivf_flat": - index_type = IndexType.IVFLAT - elif index_type == "ivf_sq8": - index_type = IndexType.IVF_SQ8 - elif index_type == "ivf_sq8h": - index_type = IndexType.IVF_SQ8H - elif index_type == "nsg": - index_type = IndexType.NSG - elif index_type == "ivf_pq": - index_type = IndexType.IVF_PQ - index_params = { - "index_type": index_type, - "nlist": nlist, - } - logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params))) - status = self._milvus.create_index(self._table_name, index=index_params, timeout=6*3600) - self.check_status(status) - - def describe_index(self): - return self._milvus.describe_index(self._table_name) - - def drop_index(self): - logger.info("Drop index: %s" % self._table_name) - return self._milvus.drop_index(self._table_name) - - @time_wrapper - def query(self, X, top_k, nprobe): - if self._metric_type == MetricType.IP: - logger.info("Set normalize for metric_type: Inner Product") - X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') - X = X.astype(numpy.float32) - status, results = self._milvus.search_vectors(self._table_name, top_k, nprobe, X.tolist()) - self.check_status(status) - ids = [] - for result in results: - tmp_ids = [] - for item in result: - tmp_ids.append(item.id) - ids.append(tmp_ids) - return ids - - def count(self): - return self._milvus.get_table_row_count(self._table_name)[1] - - def delete(self, table_name): - logger.info("Start delete table: %s" % table_name) - return self._milvus.delete_table(table_name) - - def describe(self): - return self._milvus.describe_table(self._table_name) - - def exists_table(self, table_name): - return self._milvus.has_table(table_name) - - def get_server_version(self): - status, res = self._milvus.server_version() - self.check_status(status) - return res - - @time_wrapper - def preload_table(self): - return self._milvus.preload_table(self._table_name) diff --git a/tests/milvus_ann_acc/config.yaml b/tests/milvus_ann_acc/config.yaml deleted file mode 100644 index e2ac2c1bfb..0000000000 --- a/tests/milvus_ann_acc/config.yaml +++ /dev/null @@ -1,17 +0,0 @@ -datasets: - sift-128-euclidean: - cpu_cache_size: 16 - gpu_cache_size: 5 - index_file_size: [1024] - nytimes-16-angular: - cpu_cache_size: 16 - gpu_cache_size: 5 - index_file_size: [1024] - -index: - index_types: ['flat', 'ivf_flat', 'ivf_sq8'] - nlists: [8092, 16384] - -search: - nprobes: [1, 8, 32] - top_ks: [10] diff --git a/tests/milvus_ann_acc/main.py b/tests/milvus_ann_acc/main.py deleted file mode 100644 index 703303232d..0000000000 --- a/tests/milvus_ann_acc/main.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import sys -import argparse -from yaml import load, dump -import logging -from logging import handlers -from client import MilvusClient -import runner - -LOG_FOLDER = "logs" -logger = logging.getLogger("milvus_acc") -formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') -if not os.path.exists(LOG_FOLDER): - os.system('mkdir -p %s' % LOG_FOLDER) -fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'acc'), "D", 1, 10) -fileTimeHandler.suffix = "%Y%m%d.log" -fileTimeHandler.setFormatter(formatter) -logging.basicConfig(level=logging.DEBUG) -fileTimeHandler.setFormatter(formatter) -logger.addHandler(fileTimeHandler) - - -def main(): - parser = argparse.ArgumentParser( - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument( - "--host", - default="127.0.0.1", - help="server host") - parser.add_argument( - "--port", - default=19530, - help="server port") - parser.add_argument( - '--suite', - metavar='FILE', - help='load config definitions from suite_czr' - '.yaml', - default='suite_czr.yaml') - args = parser.parse_args() - if args.suite: - with open(args.suite, "r") as f: - suite = load(f) - hdf5_path = suite["hdf5_path"] - dataset_configs = suite["datasets"] - if not hdf5_path or not dataset_configs: - logger.warning("No datasets given") - sys.exit() - f.close() - for dataset_config in dataset_configs: - logger.debug(dataset_config) - milvus_instance = MilvusClient(host=args.host, port=args.port) - runner.run(milvus_instance, dataset_config, hdf5_path) - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/tests/milvus_ann_acc/requirements.txt b/tests/milvus_ann_acc/requirements.txt deleted file mode 100644 index ee6a8a11ff..0000000000 --- a/tests/milvus_ann_acc/requirements.txt +++ /dev/null @@ -1,9 +0,0 @@ -numpy==1.16.3 -pymilvus-test>=0.2.0 -scikit-learn==0.19.1 -h5py==2.7.1 -influxdb==5.2.2 -pyyaml>=5.1 -tableprint==0.8.0 -ansicolors==1.1.8 -scipy==1.3.1 diff --git a/tests/milvus_ann_acc/runner.py b/tests/milvus_ann_acc/runner.py deleted file mode 100644 index ac5fc0a6dc..0000000000 --- a/tests/milvus_ann_acc/runner.py +++ /dev/null @@ -1,162 +0,0 @@ -import os -import pdb -import time -import random -import sys -import logging -import h5py -import numpy -from influxdb import InfluxDBClient - -INSERT_INTERVAL = 100000 -# s -DELETE_INTERVAL_TIME = 5 -INFLUXDB_HOST = "192.168.1.194" -INFLUXDB_PORT = 8086 -INFLUXDB_USER = "admin" -INFLUXDB_PASSWD = "admin" -INFLUXDB_NAME = "test_result" -influxdb_client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) - -logger = logging.getLogger("milvus_acc.runner") - - -def parse_dataset_name(dataset_name): - data_type = dataset_name.split("-")[0] - dimension = int(dataset_name.split("-")[1]) - metric = dataset_name.split("-")[-1] - # metric = dataset.attrs['distance'] - # dimension = len(dataset["train"][0]) - if metric == "euclidean": - metric_type = "l2" - elif metric == "angular": - metric_type = "ip" - return ("ann"+data_type, dimension, metric_type) - - -def get_dataset(hdf5_path, dataset_name): - file_path = os.path.join(hdf5_path, '%s.hdf5' % dataset_name) - if not os.path.exists(file_path): - raise Exception("%s not existed" % file_path) - dataset = h5py.File(file_path) - return dataset - - -def get_table_name(hdf5_path, dataset_name, index_file_size): - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - dataset = get_dataset(hdf5_path, dataset_name) - table_size = len(dataset["train"]) - table_size = str(table_size // 1000000)+"m" - table_name = data_type+'_'+table_size+'_'+str(index_file_size)+'_'+str(dimension)+'_'+metric_type - return table_name - - -def recall_calc(result_ids, true_ids, top_k, recall_k): - sum_intersect_num = 0 - recall = 0.0 - for index, result_item in enumerate(result_ids): - # if len(set(true_ids[index][:top_k])) != len(set(result_item)): - # logger.warning("Error happened: query result length is wrong") - # continue - tmp = set(true_ids[index][:recall_k]).intersection(set(result_item)) - sum_intersect_num = sum_intersect_num + len(tmp) - recall = round(sum_intersect_num / (len(result_ids) * recall_k), 4) - return recall - - -def run(milvus, config, hdf5_path, force=True): - server_version = milvus.get_server_version() - logger.info(server_version) - - for dataset_name, config_value in config.items(): - dataset = get_dataset(hdf5_path, dataset_name) - index_file_sizes = config_value["index_file_sizes"] - index_types = config_value["index_types"] - nlists = config_value["nlists"] - search_param = config_value["search_param"] - top_ks = search_param["top_ks"] - nprobes = search_param["nprobes"] - nqs = search_param["nqs"] - - for index_file_size in index_file_sizes: - table_name = get_table_name(hdf5_path, dataset_name, index_file_size) - if milvus.exists_table(table_name): - if force is True: - logger.info("Re-create table: %s" % table_name) - milvus.delete(table_name) - time.sleep(DELETE_INTERVAL_TIME) - else: - logger.warning("Table name: %s existed" % table_name) - continue - data_type, dimension, metric_type = parse_dataset_name(dataset_name) - milvus.create_table(table_name, dimension, index_file_size, metric_type) - logger.info(milvus.describe()) - insert_vectors = numpy.array(dataset["train"]) - # milvus.insert(insert_vectors) - - loops = len(insert_vectors) // INSERT_INTERVAL + 1 - for i in range(loops): - start = i*INSERT_INTERVAL - end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) - tmp_vectors = insert_vectors[start:end] - if start < end: - milvus.insert(tmp_vectors, ids=[i for i in range(start, end)]) - time.sleep(20) - row_count = milvus.count() - logger.info("Table: %s, row count: %s" % (table_name, row_count)) - if milvus.count() != len(insert_vectors): - logger.error("Table row count is not equal to insert vectors") - return - for index_type in index_types: - for nlist in nlists: - milvus.create_index(index_type, nlist) - logger.info(milvus.describe_index()) - logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist)) - milvus.preload_table() - true_ids = numpy.array(dataset["neighbors"]) - for nprobe in nprobes: - for nq in nqs: - query_vectors = numpy.array(dataset["test"][:nq]) - for top_k in top_ks: - rec1 = 0.0 - rec10 = 0.0 - rec100 = 0.0 - result_ids = milvus.query(query_vectors, top_k, nprobe) - logger.info("Query result: %s" % len(result_ids)) - rec1 = recall_calc(result_ids, true_ids, top_k, 1) - if top_k == 10: - rec10 = recall_calc(result_ids, true_ids, top_k, 10) - if top_k == 100: - rec10 = recall_calc(result_ids, true_ids, top_k, 10) - rec100 = recall_calc(result_ids, true_ids, top_k, 100) - avg_radio = recall_calc(result_ids, true_ids, top_k, top_k) - logger.debug("Recall_1: %s" % rec1) - logger.debug("Recall_10: %s" % rec10) - logger.debug("Recall_100: %s" % rec100) - logger.debug("Accuracy: %s" % avg_radio) - acc_record = [{ - "measurement": "accuracy", - "tags": { - "server_version": server_version, - "dataset": dataset_name, - "index_file_size": index_file_size, - "index_type": index_type, - "nlist": nlist, - "search_nprobe": nprobe, - "top_k": top_k, - "nq": len(query_vectors) - }, - # "time": time.ctime(), - "time": time.strftime("%Y-%m-%dT%H:%M:%SZ"), - "fields": { - "recall1": rec1, - "recall10": rec10, - "recall100": rec100, - "avg_radio": avg_radio - } - }] - logger.info(acc_record) - try: - res = influxdb_client.write_points(acc_record) - except Exception as e: - logger.error("Insert infuxdb failed: %s" % str(e)) diff --git a/tests/milvus_ann_acc/suite.yaml b/tests/milvus_ann_acc/suite.yaml deleted file mode 100644 index 7e772649a6..0000000000 --- a/tests/milvus_ann_acc/suite.yaml +++ /dev/null @@ -1,29 +0,0 @@ -datasets: - - sift-128-euclidean: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - glove-25-angular: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite.yaml.bak b/tests/milvus_ann_acc/suite.yaml.bak deleted file mode 100644 index 7736786d03..0000000000 --- a/tests/milvus_ann_acc/suite.yaml.bak +++ /dev/null @@ -1,11 +0,0 @@ -datasets: - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_sq8'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [256, 400, 256] - top_ks: [100] - nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_cpu.yaml b/tests/milvus_ann_acc/suite_cpu.yaml deleted file mode 100644 index 44caaeb91c..0000000000 --- a/tests/milvus_ann_acc/suite_cpu.yaml +++ /dev/null @@ -1,19 +0,0 @@ -datasets: - - sift-128-euclidean: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_czr.yaml b/tests/milvus_ann_acc/suite_czr.yaml deleted file mode 100644 index 7e2b0c8708..0000000000 --- a/tests/milvus_ann_acc/suite_czr.yaml +++ /dev/null @@ -1,20 +0,0 @@ -datasets: - - sift-128-euclidean: - index_file_sizes: [1024] - index_types: ['ivf_sq8', 'ivf_sq8h'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [16, 128, 1024] - top_ks: [1, 10, 100] - nqs: [10, 100, 1000] - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_sq8', 'ivf_sq8h'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [16, 128, 1024] - top_ks: [1, 10, 100] - nqs: [10, 100, 1000] -hdf5_path: /test/milvus/ann_hdf5/ \ No newline at end of file diff --git a/tests/milvus_ann_acc/suite_debug.yaml b/tests/milvus_ann_acc/suite_debug.yaml deleted file mode 100644 index 44caaeb91c..0000000000 --- a/tests/milvus_ann_acc/suite_debug.yaml +++ /dev/null @@ -1,19 +0,0 @@ -datasets: - - sift-128-euclidean: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/suite_gpu.yaml b/tests/milvus_ann_acc/suite_gpu.yaml deleted file mode 100644 index e83433ffb9..0000000000 --- a/tests/milvus_ann_acc/suite_gpu.yaml +++ /dev/null @@ -1,19 +0,0 @@ -datasets: - - sift-128-euclidean: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] - - glove-200-angular: - index_file_sizes: [1024] - index_types: ['ivf_flat', 'ivf_sq8'] - # index_types: ['ivf_sq8'] - nlists: [16384] - search_param: - nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] - top_ks: [10] - nqs: [10000] -hdf5_path: /test/milvus/ann_hdf5/ diff --git a/tests/milvus_ann_acc/test.py b/tests/milvus_ann_acc/test.py deleted file mode 100644 index 44ffd53051..0000000000 --- a/tests/milvus_ann_acc/test.py +++ /dev/null @@ -1,33 +0,0 @@ -import time -from influxdb import InfluxDBClient - -INFLUXDB_HOST = "192.168.1.194" -INFLUXDB_PORT = 8086 -INFLUXDB_USER = "admin" -INFLUXDB_PASSWD = "admin" -INFLUXDB_NAME = "test_result" - -client = InfluxDBClient(host=INFLUXDB_HOST, port=INFLUXDB_PORT, username=INFLUXDB_USER, password=INFLUXDB_PASSWD, database=INFLUXDB_NAME) - -print(client.get_list_database()) -acc_record = [{ - "measurement": "accuracy", - "tags": { - "server_version": "0.4.3", - "dataset": "test", - "index_type": "test", - "nlist": 12, - "search_nprobe": 12, - "top_k": 1, - "nq": 1 - }, - "time": time.ctime(), - "fields": { - "accuracy": 0.1 - } -}] -try: - res = client.write_points(acc_record) - print(res) -except Exception as e: - print(str(e)) \ No newline at end of file diff --git a/tests/milvus_benchmark/README.md b/tests/milvus_benchmark/README.md index 05268057a4..73c2bc6083 100644 --- a/tests/milvus_benchmark/README.md +++ b/tests/milvus_benchmark/README.md @@ -1,23 +1,42 @@ -# Requirements +# Quick start -- python 3.6+ -- pip install -r requirements.txt +## 运行 -# How to use this Test Project +### 运行说明： -This project is used to test performance / accuracy / stability of milvus server +- 用于进行大数据集的准确性、性能、以及稳定性等相关测试 +- 可以运行两种模式：基于K8S+Jenkins的测试模式，以及local模式 -1. update your test configuration in suites_*.yaml -2. run command +### 运行示例： -```shell -### docker mode: -python main.py --image=milvusdb/milvus:latest --run-count=2 --run-type=performance +1. 基于K8S+Jenkins的测试方式： -### local mode: -python main.py --local --run-count=2 --run-type=performance --ip=127.0.0.1 --port=19530 -``` + ![](assets/Parameters.png) -# Contribution getting started +2. 本地测试： -- Follow PEP-8 for naming and black for formatting. \ No newline at end of file + `python3 main.py --local --host=*.* --port=19530 --suite=suites/gpu_search_performance_random50m.yaml` + +### 测试集配置文件： + +在进行自定义的测试集或测试参数时，需要编写测试集配置文件。 + +下面是搜索性能的测试集配置文件： + +![](assets/gpu_search_performance_random50m-yaml.png) + +1. search_performance: 定义测试类型，还包括`build_performance`,`insert_performance`,`accuracy`,`stability`,`search_stability` +2. tables: 定义测试集列表 +3. 对于`search_performance`这类测试，每个table下都包含： + - server: milvus的server_config + - table_name: 表名，当前框架仅支持单表操作 + - run_count: 搜索运行次数，并取搜索时间的最小值作为指标 + - search_params: 向量搜索参数 + +## 测试结果： + +搜索性能的结果输出： + +![](assets/milvus-nightly-performance-new-jenkins.png) + +在K8S测试模式下时，除打印上面的输出外，还会进行数据上报 \ No newline at end of file diff --git a/tests/milvus_benchmark/assets/Parameters.png b/tests/milvus_benchmark/assets/Parameters.png new file mode 100644 index 0000000000000000000000000000000000000000..2ea108dc97e8079310c4358ea82b92e6e3075729 GIT binary patch literal 50835 zcmd?R2UL?;)HVu(f}<#7p^1PaR;q{;fdpqnP*Jhaq$N@W={-OQppJz;fPi!b5$U0Y z8VFJ%H39;mMrtUbgpvRu$$x?~|M#uC*1hZAe|_Iw|6O;jS(6aRdC$Ag-p_uXz4vR# zy}P;x_aEQS#l>~-<_+x!TwHtMTwFZ&_wj;HoVf*;xwuYp-PHcWz&C|5x`=rD8bxN4 z*q49B@aLSrYeRpp_D9R9+rc`Ia-@H#h4lJb5pGGAnJIend z8Q?#c#8v>i;Nr4>!*6oL8=cR>6BYulB$3$Z}q< zz7T2{dSVqg!3BAtN8zT37hMD#OySG=_4UcF*TEs8WT%3%aQu1`G>_w z`qJhSqF$^BOEbNF=T5;C-YjN(d^|V(N^+;lR8{-*>U6td)MGgO{Wu<-#;j-4*%N$P zn|}nXC6mGlfwWS00)*Zqju<&C$spnwRGVA4`+=gxUBuspHc|d0CzHld3O>Loh)e#bN38A-5jS4cS zQ59-e{%o}t5LYjg#jj4q>i2KZeH3{poyn?`gLdf=HA~eC@$6L1^>5dx{bsIm?RI-O zYY$b6z>kJ#JCW)jV?k_Y+)ibfRj`F+{m!&Py9A!J(@A=(Nt@|RE_R=6iNP-ol{>cL zsWZv0ExORvXj^f3_NPx)%VXH~)#WK`NruVJ_CUSmD*2zKMLA1ztmDqy-iX`z>aB#6 zFCwQQ3n}g?ntmly#gmhhwvvq00QQf>F?I`_UlSf;e12LbEcTeK(Lz57G(* zrk9;7<+#yBXN!aA5baZG$fc@(7jzAn6tj(*MAS1ZTCHLJb6F)(5FZLLCUTruY#p{; zV@lv+EVEh5#g7|b*|HHmy}i9}>32EtV&ZJH!Byk8Hskx_8DB1KPCv;1J{{}i>>MEg z-Graf^0DZd^Xeloi`d&mg6WRIa(#n?k39IzDqQ3k#QM#!`t!&Ry(+EwEWLz`f&%A3 zKB7Xt^2Aa^MyE!Y{O0w}mJNPqEqm}N~ zBbDxx@loR%q^+)uqCX3iaWfuOmB7`dp=<&6ioj$lVk~EKCulo6C{)f2V-Ubnt)oVm zv1&P2zHN4t$Eld(JygD+6+S~4T?V#wiwD{{|2Q~x8OEB1wMjHKyx`N5&`?%3^m!Yw zIsV#pi>aRuweaC$b=W~mKXTKWLXHZiGl27H1~P_8{hF9`)_S%eZ1QOOW|Lgd6V7!$ z6;yGnWNd$ad*iefK2p4W=cbV`4<$f1^27xsze4?vN2@gnMIW_g)~JEgukl%w_QTR> zv`~6d0U{VR{!&;PN@tMRfunfh=%@wNdv$9^4(0N5l()Vsz3OAd@9h3yFn_>~p6fH^%_=;Ck%z9m8|v%Fjx<4Tf%u~ri% zUu*3=HplvVk4+l4b1@cOgK8*>Z)^U%oO(9_Md{Aeb)CZoZ8l2Aqp(NM-8Bd(^5-M6 zStK^HnJhoUD&XPa5rmOP5hQFD;UL%xFx|DT@I|EP}9`C9lDp1TdUJ0 z6^%E}#vZ=!M*(JGHV6dXqgH7#1_|@fE*`!^1_7yh3G#B=o;=|47wzsGLJKQc=5JTm zAeIZa19&K9rFMiCGL&AG)VT2Fkhr?nFdxy>dd33*pr>E+lUy2u8Oz2gNFT!>AXv!+CW9B{|IYIM2(9t6pMQs+&t6Tc^?Oq>Q3@$Z> zT*g)|*D+lD-sgJ<^_D|%bxhi@J6;@NAg+_3jtd0-Lk-an$e?%*4-ZdHG>0h@hU<6Q z>)%4Uv`-yoZ)vf~)m;$I!NviXp{>vLQjzs^q#$e!0CyUqsK=w9!NoB}C7Z=axO{KS zd%$ABCpa}JNg8Q5$E1@a2MR6n81Ebpq7NVD6OhQ?+))UmhDEne0}tv*RHC-(2bN5% z1+B)W%Pzg*yEE?2ZaFpj`n;V<;oCDvg^q--|wD8Sg!qy%hmn@Ns z<;41+pk^7^KM?4m#kne)pMy9f%qZZi^Xi2gnz$#WgVkNr80X=Mx$N{vyL6NaYgK`L z$!BTgX>i2cA6^Un1!tnz09NZ}tqbjfBi{J3ne@pIuO&<|1fOLry#<)Q;$u1=XZ7Cx zX093l)SIt^HeXjqIYI=`ocK5O^bG*9)MhE3csCokzV~%rBAXG!cCB6ca$wDWsfyfI zhp@A=1L)~H{&I6YTg!q*C1|k&TCD+Wvg;`BC)C{DPi=t%sTYoojkP5Vc<=)|4-~bLXibNvk+A*hW!Si~BcKw2emvUu=T{|*-t1b6dXZ?>?v`GR+F>brNJ{h(2jT9zA1 z?A3(&aE;~KHH-jydS|$vX*Nh_xw04Q<1~F79C9hbrlTMmdDoL%0r;1sGpopHGK|gX z6L6_hf#5>dkw}Id+sto!E?ambkaO$bF*AI`gJLR8KYqXgGG;DKnila;=yh~#E@5F| zA#cGmp=Q6=8D65~&JR9Y5`(>y&VIN`q>mwsDAtleJJf2@vL0;vLF<}iz)C}C5Ix>8 z7#qm+2X}rFy4@WmzeQ?C)UBm{C9iq^#I{_sHIS9l**oM(5d8%UmHi7`!vLi={Ei+y z>Wx|gDdZ#w_R0w!N~}Kg5e-X7gP5%nCE1&Jkn<>|g#Hant*uu5HYnS;e%r8_EQ{wt zf;_o7AzTkqpV3l1n^hgu&$&5+3Kw1?h!c=_B1kM+g)%tWLulafVIZsLO?X1tTZ92- zmDH&^T6s~yCKp3DXkGWrat#2-5_(aASD#Dynx<4bA%UsUOCrHG{PcDc_rx%X<U2t7p{IhYeUFV4;2$|`|mDd82g$z$! z?VVX&Ez@pp8i~y$wbv~x5i}ptsLsyLXQEIm4SbGuguzm~%&e@esj9}tM&YtU-rb> zz$6!=M^C{J)ukV}8e6S}&~D0km<=q#tOP0jz=qaLbsVgZ}r@ zImyR>_26S--9~|{VZf6A2{p);R+kXbK*hxdhSU z$8*8D-|K!Eo*r+=7`SWT-LKeQ|LyOg(7O z2{X@tf_QCT_2e*6lc;{mZk+CNtvVW@S$=xPI2TeKcFB5kj9xcznTNvRY#Ra0*7Tj5 z9GhE#(B~!c9zvIo(Lv(yPOs0@jWR+N2*U#02Lp{bcf8PrSRp5^T{QdvCj7zF5A;pUFRHJek>Z_;dC$y;#v z!og5MSW&-W#Bhac6yP$30W41Tu4F9LB)bGND}opmNq9i0*p(9u9qpZYgbhhPY5+iQ@X=heIhhzV?!=unYua#v3j zQIk6S$nDOZkyFjNk@8?>ex)cZM#Bc0Lm`R&{^;rh+% zeUsZ;grGpPNNeRoXnd4p(E?Et2o~R&M7s%)7sCJrY~s+n>nNJeL9{~Mrqv)_!(-}3 zt96li)dN~_6oh7SEzuu|NA)!i!?xOta|2t=-RUEq`sQ^3 zTbLA%`VKC47(IyA*J65gP9`NKh4M-G!dsU&fbV^r<0XQK^qq>t#WE255{z84AjxuY!Gm1d;}KeG8yjg<6;1H^nIgQ`|x2c zkX+-}EkP97bt+_$mn~aNL*Nm1-+V8W=F95jmpRRtw`1Zw;u|CJ+Z=>02Fk z@rxuq7!1ZC8V%_Z;8Jju3DD^Aa%NTluN4&)-SzN@wV6&ew;#9+(%N(aEb@*0JjWg! zqPf!`xzmtAU~t$;Wx&>|vm~4Z)PKn`fNWGAWwJdY9$-&x0*SID zTh`ABnjRGuxSn|?irHpdS33;!KOiD8Fn*iYw#6Jf>J+f=5iAx<9V7&*0BW4Z!Fj9RPj|Y<#v}j$Q$cA(M?EzXz*NEdI42|I zKjJm%57%N%YRQWlfY822yEXga!-tZe`ASgD1N;qFzx01YmFvHtQ1-vs5;E#I%Nid! zd05>Ej<*5%42wB^lk;D{fy^b`-8*-hL9t~52*^?%3eZ}UfX)2#2W3yH7kMb(zZ*<} zI*7;gE$||!#r@AOzkhc5M?d;+IMf0SDZ?5+v%Bu;oaoVhZrb+7g8N)gR;AZmPiT4d|x-4*MtYw+WFhBgWz?_el86OdBm4t=;^HD)ExLi$- zq{%;*cQNKQ`FG8G|DSlfU>6KQoQg?SZE1;69;#LLp^ON&LSDG% z87Wt`zt++R{WiT)G`uwrokg_t4-OCy*U&yr#3ho`1J68&S`HO=en~EO4oj_}HH~*9 zN|QIdYGa1PT6Hnk&Bn1RuA@R4h~<#)Aus7&5aDw^u7&=2ubG2cwXA~l$pdyJ?p|!` zVn;HjYOU1Uj1q8K6#1a2a{JNGyQxUw#(c4s9M%jvp{b^EC@sv%flfJTQk7avZ`z@j z*qj()+)5$Vm(JD;M!l$DK@ac|oujlQG-Mz_3)b%IO>j!CZ_o(lh6KdIKk}|*7#?Au zg~8ZRg`MO)f-Tk;NNrc@e#?%6NN6~uPGT(FSK=J5{p0QiP#&i$e7re#jnX0Ez&kAS zC$3I;S3JWU<(iDsOFG~4{IYb#RxMLVYF68 zFnd2i!`nBAGPq``aDx}PhKxd zhp4f4oKUiYiArRkjy&Dxn|?gE9zIbsWATR>LjtYm$J{1RUw&O88LdL`O4*YgRZ8ya zp0bCQlGsW_MPG+zcEhdRqspl=#q&6sQO2}Hatw06!B33Tt4bhk99!Kgya_VBbTLRm z2)20=HaxjjsEKtO|1;%yik8HZx+N-jl(yL&1IvWXGv4x}hiO8*Q-5YZIN)*`FcMxL>%)YhJHQq0tc3n-0gQ0viO78Pt&8U}|U29ofE?d`BD`0S{~ zLSLw=%a~BqX?n|#ZsQ4qm(k|`++Z9x=63S1Pty`4N6o!@#4$DGC?7&$(<-|SXugWY zP6JW6D--2n8*JelXP*VijB==kfqr6g;+4929?An)kUM$P>jbH$bakKE`gRRXLZLe1 z`A>f|fGzk27p$K#?^72}#17x$tuD|&;H{B|bmoaN(Ki?KgMvrX3oag#Uql_hf|Cze z?2OVBLeeNX-3@9g8YM=%``<{Ptm;Y)JQFDqpy{({pQqkrB;)Z&Vp$g(J6>#`&Oo{qtT?$9=!mjnJAnLVSAm8%xVh~9d7At zRa^4e^V1Hd+PuW;wJlX0V}2ETh&#lPy^~{!YVjkrQ`|l|%e`N@X-DCeP=QT)X1 zRy#G}!Aaf8mGQ=2amf93aF}&}IffL$*k75qPC!yZ4IHX7i{6ec&_{AIqn(n2@^eZm zvi|;Ql{ULzizdgEc1mK9e!nxHl-XtTG-zG+R9ZoW3XWq6kj~WvUucAqkYQoJ%wE`#5zE2+XU5C%0G7qxohxD~I4xj01^IB>_=YU3ekLFBfN)n+ z2qX)3#AvABcZWC^Lr55fEmaL$1^LVs)xSgH(vY^xJ9+TDazDfg%XO0Q&lL;k;M@aK zZ=VQs(oKwZ(bbG^YntmA+p3qScempg8ne3(`>p&h`cqI$-G#s+eyEW<&IxIw|FLbv zcmvQM_wf!Yv24TSn7e3XvE3Y?;)Ch|H;BFeZ#@YR4{Y9eZ^ zp~2t;#B#iJbag?Y&@;P~+C~NWWd(jh6D;)U5Waiz+$qnYm49O1jKC%H^O83Yw*CM4 z?L4S)12P+Pdeo}Q+@(3{kC-H7wyHsl3uO2I*kS0;_IAGT|9Zb0(D=()pjOrdN|;i~ zojPY@<0~>O|D6Sh-i=n%I8pesPoAKz@%2e5rw82p@__IJ(0!@9zwXKH0E@(dMYL6t zBS2wXLEf@f1oW%rEvkP;dzrKn?w32ZKqHO$FBEM4cp)$Wn&ljM3aa8ip(pPWkL)*t zl0hePpvVeAb&6A@UKl9C13@+c+Q|R>l|1LKE+yXP^cCF~hsrB~?&dV(*MH|!>z_Una^tjVrfM>5^L2OZ zjbkj$!y$#y@ABy{&%Msfo2`wyke|9l(!$@Jm`h|(QgcL# z=-AiQPhffFh|3-?*A)9nMhvXkaAs!SIe$JKE7^+et_4$IRbZ&35YRW}N}dgkRu?9= zju-k#nOg8rroPs8t3=y}Oj!vLM@;xPFaxUd*>Ty!E>Y)m_|}%J`)l5pp|&qVR;|e@#nF;&D1vJukXybxEgsMyn`|+h^Q9 z`NrvtGgmwXpB$`m;`b8Ls!&#sWh$y&J|Q*d70#b!1&6tMl?!Qof^6%wWBsZ3h@vm% zx<$kbvYl>^d=acUvkZ-vV{gwjR6VP9yx1cc`PgGd~_DAzkWgHfhCiD5eZg&#uv zf>cy-K228>*P zag>K*!|&a75bW)6lDTB8JX@Eun!RN4MP>E?l zV%v2qFUQOo`dcQy`=7Rw=5f-6xt0mKT)Ve$ir5AZ< zkMp&0puDv@$ok@&_C}k0PqsGI!KDmA3;nTb+$!%8+-WznbY@uj!Fs7#7P&k`?rCsm zdPMB&?F=LXTl}>56R~V=%ok10@ufzXYqL6ix9jHE{q&X&0clBBs5i`MsO-WLYChtt zF0`vI;QV&=a+qAQ${jX+akLvTJFR9mW0xG1skJm(c0t8?J~BaON29ZT%WI_K=|iWX zb+=mIK&s@g!23-``O$s6#1B*vG5aEv*z?xt*)(j7hvoy&g`X_F=A0pAD6-E}aj5r_ z#cpUWBkCB8AMSWlgpD(;}0)`kcFOchpG) zo_7d+G+I*f4efZv?;xnt0Ks<^JYrJcjds_Ka3;H78}*LP9HD5XZY9eM6xjN2$Q%{?h)#~U9n!L`B-VzaeSF=pn8P>?>9Jh;eFhoqZn%vUE z#}H@^J4bq>Y8iATQZqGtF&Gcl6@H!sdiD@vn600B)nq9q| zD)v}15IQFBZ@i1J@ZbXAP~n6z{jweBeIANE7MitvO1K&8fSIWZRXy| zVO|3ri+tOel*K!)H@ok>n+vV;IYC!I^rm@{M}lNYdjgvhKmDtdfGn zwLytd3ney^ypBT1)Vp_cnYk8pD`^)05HX(fbG8bTgcI;#O2+ljfYV*^sN3?EcU^L> z;CzftZ>+fZEZ1XR2 zg1p2r5qq>!IVoX&ORlK4Zswwuy~2bU7dn0vV^oi5=o&q_qrNwai>5Y zpn_2=TR@O^5UU}FLPh#*LX(+0#q6O|4;G9j=>TgaG9AR7^@?9QoX% zK$H%&SNAu4vZPS`qaAbhLARpPHy?$aBSpFCnkA)~8Qho_F)_GtPBZEGkF76q`!L$8 zHq&}7R{9>{>NJUhtG^k$)mFYe98a|eAUli_G zTg>efXS$Q%hCR%r-wpb80Hu$;PhI{tNS4f}0!>`_zO%W#RjxOxW_j+Np9z%4LDYqlZp$0FYTj_GwD8%vUc2~C8Ub7#$bR|JqjE0?mX7D3Mjfb=!(;_CfWQ<+ zMqzxRNJqcO<8!*1_1MjR-v4QA2u=rTxm<(rpBjk?pPbg|+7Y9EMQ?L6_%uGtqPIif8d5~12@;k27aDEG&h)rbPC7+s6FgCzQ}hAR$blGGDp{c+k}LokYA$Nx#! z+N-SYl64sKEc&{M9uLJqz37e;6+{JV@meX*+^0qT=Jw`mb|972*|k%?nmPxMmVson z9OO5Kg?qZ52wjr%l$u{_4)Udar7&FPt{hoYA?_qo16Ic}22?nT00$VG#dj_G_MCW}k;7oCC`L5l7b-yvaoFd^1=}Lzj5l9$WD7OzrZQiJ5 z&P4kYNaovqS zi`N?NcJb9KT`MoAo;Ua;j|Cp5KUhw+fjc)&sC%7H)AG<9Q-f^^uhO1i+lP}GVjSNecB2J#y*p++V2^_TrsyL)sg@D06l)hj@UbBFo z#vzEwW=THYQlZ*mWY=?LBe>(Iqm#p1ThF}p%-l2YDN}LsYyH+b1q-Y2(h^swb64@$ zdh<0kN);BQn6@-_38^q}&6QBETC$6%Ci{70+fPa+j>Vg8XXFf4>yGBdSC0%q;A_;G zHAr#RR2X_D=z>wnR9M+AT;O^{;E#l=sJFu-BgZ@v6-hU=QSsnrwo1=U-h=6DRx-y1 z6@A)K0=gyad$l`0gUF*lwbdCgrG3Zt!Y)C;Ek5~g6k7imA`Ym=5Fj_jc1ivxI7deU zF$)G|aJ#m=yq6<9&z?PN6%g>#x<5YwV_JtnH?k{TOY-5vhcYNNaE@Gt|4A+@LZK3% z6ZvSaaaU}9{a0=`HEG+F4t}+{^dIt<^VR>-S@!?UT6F(QMG6!0YV@LGgm#9qf9X~n zR0P_pqHjwbfi)xB8o?D?Q(U;h+PNXePZBhY*P5`Tm}E?p-koI zH((ice_Hcb6Kc`lQTVIKQ%9T`ROSHuKH9DEcO=!X-aReqwyWOh zBhKAgM%zGsM;G2WeWW`&=`%y5^cu6fw@(ojUa;6bI4Z_7M}5bh-x86ukJ8SNqeVf=>pm2Jy~BUq?$lv9Y~mMWK~vS&X8V#e zM?$o{4&8}ou6fbd-4Rj8*nZHnI((T$*-ArpQN`zI2kop`dvnVPW-d-@w9dM_r%Z>` zGiri-8{!JLzS4}V_x^#^YJ7)~t#Xp~uNlh7T0xXzYqVZjSb~`hv^;Rgnb`Cp{QsgHkBs?M zB_VQ2`_vIRNOT%RbnELOFEB76zUYZ3#-9kOLgWVmkf>t!_w*l5O;xf+ojM}Uvj%nC z0@+rpv5Fg`+Na8_b589Z9aYwdJZD;MT{&_Hj7j5?H~`V5oqO?{_d0%-@;)xCxMA-8 z#crs`TQqk`M zqXvszpWH|h6@3_;_%+V_@fBFNr#iivGIJtUvAW3G<+8aNZfwajS}ejw@6?fyLXk%o zp))U0;EX%E1=)siKn3?83f7Rjg2Kjru|0NZcotJdA5c;KW`ym(>O zc*j?wXm33gs;Z8m+1}#q+#%M@$q1S;to(J&3b}n;b{1qje_d$0H}HJC*#3r-q=}nM z@GbK1N3jJj38o%uN2BLIYvRVnJQV>$scP`tuFpn(BHUl5Txn~*L0?A?CR|{8GY8!< zQihfLF>QYBQ9AnL^F!TxUu#`a{|$vjDPB}por*t%4r7W69Yl*Ou?2!N^vy()%^0Q1 zue>>%KF-;63t#r@FN23&>PCE3E+snMo|2l;!LLeLHEHwv+$Le*a80X09enHfY>Mg= z`JJ`s*;}Vib@#(Il{c$?FUEQq%FAeBT>Zl-vTJ$F!-`*s!DX-u0OBN^99H@}}>@l`#U0meW>}1Ej5pKxrV_)K-WCXqP1f56E z7y?ZvkAnhm3-B1xoiphd!u?u%^3?P?!y_#eAJ7<4tI;E; zre8M|dZA$7-%84v-ybv%{MH^tGZ2BhE?2z^bOZWBu2n>?7Qx}4A=+gdgwg%K?}wd; z|NO{q+%mV1Fl#87IWv>7$sx7uJ8|@*R1GYNC`#o8IzfV@0wi| zl?)R2(G_&_V-gmX2OBynfl?-{J1)xvE`~JBA@eX^IweI4+tPhnCY5C8#kF`zecjx8 zyQMJn{&Js#+E08Q$*JPxX?FIUlHPkwPEvdFDqov!zxCKDEwQ^;s%`nNsvC%Np|2|D zGbg9+*`YG6BZg0RuyRpWGpvt5rLJW{42$-3Wz|TVb0dM)?#L5%x%O=+IV3<{93s~Y zi&t68cnM>;4h!2Z$K11WC(EfuM|kBbRPVQ@jL02s8+i`z*1EQv?58=f^k8EwrAlP+ zLS2u?QdoAiZrjc&zi#12ojbP}>$Ak;s51&sy6ppHU3|c}@zT-J`!r{}KrFT+;bxx( zu0`8n+o8L}dU~WUf4c6k1x#AlVD_=Dn~t!Lx}WfqG32hX$?_55usiiFbvF9z>)8h? z_5!y#t1G2^@wXRJlUXmw9i$KOV_%Q)uEcQr{O5h0RXThvwU@q5kw|){_2kYWU4-8F zz?XAUp}Kf&v!L;Te!Hnt8W)+@4MVy<4F2x*q*7Lt@7#k@`ndv#de_0Xi_F9=~2}{Wm14rb#`_e-F@COvJYLrOarFAz{eN`g7thKn2ZZj!P_TNAx&{fSu=1+^i5U(z;>zf%#eSNb2BA z$0vU+gnsx?_r>1K9JwATTN$cy*7{>b!IZaIHLp}`BCC9@dKsA{`VLE)tsO#I+hlB7 zxH^N`yLY$ggvh_vB9$Q!AIyfr3Api$`R(+-yXTPgM%4$ubPS3}CW-fw(<^NFBl?2I zRwu7TJ1MNGhs4jOU?UpV7v74)trh>X$Ht|H7vENY_25PH@w>WD0>)Q3VXPl?|AEV? zlFY{F{ZT62Zk9#&z8=K+&$Rx11_y|>A>sOX7Nd1pUbZvf^QxV=^!4l%!?WE zwduX4sP-;&yqo&FNmtO~5+8zF@Zf>#6?r zD)48)hM-fE|Jc#bD`o3aN&gM`O7E-k8W&M!&*N>vyDU|z;|ZKff4(KJ3H%`F0$2wI zxY1Ze1*wqk;bAd|#o|jKE+W$W=#MOh$(#u`BN-l9^s549x1v^BWP^=eSq0{H&>u!- zx?UUW_-hIIkBxj%uJS#NOlco0`M1EH& zOcYd%g>nY%IU{MtMV1i((4URepUto{$(=2nF*GpMz1znEPmF+}P$`$O+6%jvDx2TU zTd}Q8U?g0G(+mUS#x38!f4?Pp1eovY^?w^%GO&mu@Bz-jj-y#Py{|=%eSiMk*1?4Y z)1-0GW8kCUB+ll4J`&r>>HEbcB5}L%7W{-Gc>cTETmL7j>;Ijum)SPAP6l#Dxn@C& z94lq|I(j z&Q0ZPUhUn!`bOKS7F}z5&}>4KU3YBnAVu}tJ1&RXu;Jl0=knMVopz_Ltkr@VPygEQ zYkYW2<1f!Yn0T=``{i^Cu^(%@(%& z&RE;$?~q}2#SxD5x*Pfiz6E}un|35M=kt~|ctGJEbS#j1Rg<~?%$GIw zkC=MD9}L)Ws~omy+{k|xtJtH=_S@h`AJ82JPYubnc7N6+3AWl=zc1aaD(Ft6L^XUE z7rB{jZF63bhvA>;I7S?hKxIx_Vtvaf|5WH7sz^KggvE(1CSv_73Oy~o`Y1b|n1FO! zmtf~8R`e=CFZ;Fi`ErFAaXC+N)fv$_k8D3p!PM(o3$!#82T{)^M>w4om%8?%d%*Cr zM8$fOTVhrz;Ja3*{fB4Rlp25fb%&W{yqh%dwtq&ZuZi*;^glQ6q%iaG5%r?O8kUMy8?YhI09 z<%M`#@8G)yZ6DzpeuOs!rK7uD5!qQ%9#q3R=Wes@YwwU*Ua+#x9n^g1TmDsLP#IRC zl@lVe#8HikM$`#1@VHs&YVYeM>!tDFp=UWr9|5#kpY0Y@zR7|iNl+WF8~d`aU$9*M zYYJBQshR6~`M?$2UH(qXgM-kE60xPm!JtYu*nTc-OaX!@x97C?ke}{`cR;d#w)dWM ztE@)PrPg?qZ(ku?_n)EhoS}DG z+c1VN^-`|?I}0$=GVshv+}MC@ZRNYPtgA|ZUt z__H}!YJAv;9k*FEEKnx!(Hfmu$L(`U+0(TvQshL`n~{;L39~niB|1Z26=<8m45~(F zf0wDmRt8nD|K% zt(el84?#XzOmv)CJ5~sr@3B8}?ollLKwXOG%!2^|77x}QZ)bxU}1g|lK*5D=**ZjVlx34|8n(J7Rx>ui7q^4hMa_hm);Myee zCghoaJ1+)&i=FP|y8F4dyfj?|o*=GJ@~JRejJqH9>68&g;-ceVe7U(pRT-u*6E(0> z61%~stf<#-kKp*n%9*eO=&+45hkTg2!QeQjhnI8Vv@A76kV_dem1C(&LN?sefrVbf+*P;r7TDinvdt1Zw zMj>76JF6Ml_s!zMI@J;%mP2}a+r!fAP^0l(wOTS|FcmFx#1%TqPTHcpPqOneGU@56 zZ_#l3fn_XNwX}Q<9wD8Vm_@COd6~PkEp_LK(j6UvYGneJK7CwYY&Zb*>vOVCw?J95 z%qT?D;Zm~wx?6T$Mv!J}dtHCz*Mn&4bLnQo**lhR4RS2!XsnRw-C-CmwIkfLI4&{% z2jlI(nam5lSjd$)0$;pJ9Nt&cV7qViap`Hfjn$=&#$3#CQu5x6tY|~5>NmI>j>w!U zs#tDaA3A{iP%+OZ(I5BuufOUAG~-|VlJETVie#>*#rl)8c7=@tzUBCVvF)6t8{|6{ zsx`;&Ch*}4QU}nGK5sA*8o6A9d1Z?b^0~=F&%E|azRlBnJ_OGjZmbM$l|ZAxq!0&! zi`RJbz=6@*Dztw8@6(B5vbqcSBb$_w(I@BQJ_+H#S0|nN3)owZO2@9zWaEC!iRqR}~MQ$&=mH_HlKwLD=j3Mc7d&f-L4ncv*W;x|#nA*K>PMa^+z-@oH z{G@L6w@UHn-Geo(KY8*`=bh`dxOD2_K)_+ z*+axgXT{vg%H2+mwHDVuXqWhG#Ny<~y4epu#)c#@cjj!r8)bHX8|^nmGk7R^H9wkW za&>C%)tN7a10mMA?;wuNt^=#+^^df_s!RPOpx=WY772H zv&xnHfB59#NYR@F}pP?4A(>2@zu+3x~VOXEbm9h#E;@OhEn)w(=xn7Pm7n3 zoVGMS#Y+g`Bla%@>WBEzMJ3-6cGZP@i8y}Z=F41qsgxDDj%xuwQI0bIH+P9`?;eR(gdh!+Udm5V98E z7IJ%Q;}dC+_jbDBEh`4JVw_d1f6E>7z)f9Dt#|z~8 zE^L=#u#(9D&|EElMGrgH-oo!OS(v_AdLJgR_{v54%=B{svgrYIvYupKg--_-lPF@gLu6xC!XRxPhczlQb8STT5igW>Isx(S^ zw@|EnF{X6|j9>PDEVv~vx?Q(sbv*)4{@b(S9F8~sp_*@s#g^kYe{R~J6_53XIAV}1 zIP*8S06&^_X%DOVoe|+t;7M zY2HmV=GCP&AaP1gBtz87IqNG zJPRXT>Q*?VbFy>vL@{qZh>oLhh2yU0KR@GQ#8+@0f(B1_?jc^|ae|9J@2k#d-ER2q zy<)V9;_mOkC1aBg`r(XLvLLQ$tq&3%D8_ z2QlnSMI~SCGYhZ(Be-0-<_hA^xhnZ(yAKO&CopnZq=UH=p9$lp0OFwXafH9`kh_oNkhEGU`~bayIV22sj*xC$EVro8 z%tJl8CvW`CR^#uNzyAEko&wJQLOINBv{qfg{|8nSvlCpVfHQ@#pHjDY94uEh<~Dwj zL0Zx!zYDhKK?Bcxk>|_hl{g|8!6jxg({ST_dW4^%PaWL?_L$en${re7*+2Yydml9 z9jJUjO9wgAdBZ{g9d&84sCHmJZ>sjwrkj0mTs~gUPV~jcj@RhP==}SJ9I{1KGSqpd ze{#2F%eC@h9~wYHr=fn)>#RkyaRdOn#)$BVKNR)*aj;h>PpiwDURU^bT<6a%2ffBE zoj2ve$1%BW8tF&d6H=xR(Ds-}<@vk5Z}t=Oh!$% zM4@jj{{6*c?Z$ng3O3Gtr8)lIy^Ir5_xoo~lLtU{b(}9~txi%l&-GTebgAu6a{MMM zb5spKHxEaiB{zV`Qy%gEHUbXL)c<^yW4+NvrQd|5j>FrooelGWCm-uRTeu~NoS%KN z_SlRTDSPjW$zannA7 z19^4QYm56S82tBP@<|IKHCzoX?VkRSQ~u3o7+c_%SH`?oWb$W@m+72=(_(sdN8}8q zrf{~73pw&8aacf%q1NE-X-s}>eaY=jlt#SuZoob9kaa>?B6FWWG1x_uTXSBwLWna^ zTf+m1qdWe=7OAhF&Td3MP){|IF2+hGIw%A@4Y&RUWL>*LXZo@j{LDh}M91j}nKaqi z693HMWEsPb=Uu6fKEKQmdl1`ZG4>f^ipuW)JYU-qk^^|U-ltp5BSTtRMUfbZFMWtc z*B$fb_S{@m0mp^*q0(2O4X^!Dg$<Qt-$2~9K$%<6h^!7sE_u*nM++*&3zs1^Y(+fx6$zdT6wnCE{A$-1I1_YM9 zx%@xad(W_@zNTL|pn?U&21vI7N)ZJqQdJa{BE5GN5JHm{2wfBu1XKj1i%3mK5C}bh zB1NhKN$8+R4GASA0Yc#20shZ(-_LW-b-iESbFTAVCm(>Fy;*DRHM3^T%x`9vfSs^q znP2nzX6#e!OOMd_KD8=Q^VE zEYO7yARmAGBf1iqRUN&xbXcu@Ep#woSxlOwFI2rtxAaS2fMFP2+)S63yohnX)?Wq2 z+ppHifiWS1b6be)meB~o%5V9(`M+`miNh|}cx!4ye@;HhOE#_N#n0(eA?%}aM4$02 z!UuB?yaT0xM#4?H13`s12Zaz@GFxbK+nL=$wPH zAbut*RAg_2_W-VZ`M&vK_Il8Ry|;UNgE2j7A)7z_m9ZTiF6i$#R28FRt9T#HcU{-NhYov7wBut>M_l?CYQ~ce!ux@cjYu2OT_N4Nv)%M zQvm{{AKdi;18qJ4sI${J2qgN?41>Z>?#&_Ky?k|Sm*qg9iQ7Q;yC2VUdjtf56psJj zX8+qHocn>TdiTcuf|1~G7FuTD)gmKs8C0(XOx5iZ_ENlSnT+d++VhRfH*TBvg+tNd zd03~#LzLfv4E-%^G4|i6a_l~~JO^)JGWYy!)0I6sAGDC>yd<9+Q0 zlP1QwL5{v5C!HN)957ICw|eO+DQs1tmul_@`?f15G`V5(SI_`P49t-Pu!P&bL>lL^ zt3e&mH=eY$^aZV!_olk3~SW7Y^ebr5Kr#zZorp#~KdUMvT2)IqD z!ty`ax4lg4FH`la^*B;;h_l1InN7XWRlhk%v1GI;`jzpUF0PpNymU9702J5Pcpa#PRk_e!QyTK}ZL+dfW-vFieJNjz3l`j; zUxt}l-MQvHai@%9nD~Ik3_sg+52m;T@fnms`@1{_^9Cv@n#$mBH{@jrC1=!;hBk&? zKM4F~eErMD=FrGEw@2+Dm>rFilCA1qdwcbmUA&T-QqT~@t5f#$mP){kyuN|>l??yu zRVFQ09-qEstloh8utL6(j+${%{HYW{wsOQuGo zZ#lW{PdUrPnh2KV=rhUAiZVj`eo0ynS9OgwOsSmq7uG`einS!7=Zb`?M>9kpdW7k? zjuAaLjH0jq_6d$INy@;$fA*961LBJV&;{BeuzBp(FmKW$>DSN2h<=3ZlMZG0vM0j# zkI2fTF;NCcWgViyqIaz$?8g&?JjKr%rgIl>nkZ8!!hqdQl2G_^U00YRQ3p;pWrxjYNALcFT&Gyty^#{)(-D=ZMoprAC=4gHnjp(9z$o6A z$wKH+J>>p@uAXIP*Y%emF!(C5$hg5I*UCUO;qXG>nMTD$U88UNKB`%2;s~jmcD|;SoyY?{6Ehqc=0!GPDwNFyQo#6+n>f2CVZKR9ZLE$2L~w~xlN|E$ z?sjx89w6BYyx9!xcqc&gc|c`OZ6!1xA*)iLjTRgou^c-&mF)_vv@n%Lf4Mw!sDh7@ z*>_;zg@yU{Z^MW?OJ%OH;;&+PN?<(sSG`a3f{JZIf>{=R%{Vn6mogUWSeokd1FYwQ z!6P*t!l$=*%XaDT@-iTBo|qr)hhvNuyDJ|h@C}Z%%@*T%D4rMZV{hOb@8`PoRC8{I z%}$T4WEK?QtY5Si`(D@mGAb#`R+3c4;{1??dRLrXy?hLD$6L1c+BNr@uX0r^3twht zI77)h3yq>O*uBMj$5;L6Ps>kdjy>7c365CQoPJ<{P(p}XeknVMa(WHDs!A!BHQ z?(4Ht&_ntbzau@~&{nx3^K{VLk&jpNlsF8>QB#yG1M9i>Vg3LXuYBQoO=TTlmv3G%9W~#UH6*yynYmmS0{<+!$Vb8{c41pBYB=h7P12(bgx@3OFam7- zZYGKGJkkb-M1$hS*U)uDB>h2LHA_>2Xjj^p`qy4Zo#L?@=m&<03u5)Wh3!KdmH$Gb zwl6c1Db8y9H_qp|wz=Ucb;2^7D;wp7*dY}XYos1{qLW$Hh^;ni(`eXwdo@87@VLku zYr7HyZ#56yAY}?E(ano{OHIk2J}Et1uZir&8}fxOEs?y#8rF7%`f)p9^QP2F$Ayjh z_ZVwGQ2fmgSnlYj;kBQReq(8d#^7Ri`(c*E9a zP`_L}nnis;tE+XNZ>q1Xr8Rs0k#k~<-D9(hu}T*o^0(5<4U_R$6pdeGRoS?!Y?3&uf-j0b^oUi0}`A@3Z_H>eHUUH__SM=T4o0=wmB39ch_py1h0pbRG*D37ML0hAGT z7?#`9$9I3VB=ZA2{#IHrp4_LaHixqf#dtK(^TtEBtr<1|P}TC^1`)A5#lmQsElQmZ zbi4Dc+SZzSC{P{CX^|0*01)%RPK*3U(J9F9j3QHj7J>nm{q1&7h1H~8O9Heu&kz93 z>?J_i%vPtx9WDSh!Wdt^*i$_L8w{}9Mr7^&v0x7W_q%&saYplh<*j;17~qsCM;Kq8 zO4P|eB?D~aw+gjQ?vw%UeRwXK)WvIhLYi%|?X7 za&LDInn+|DDN2=Dm_q9MzeZ77@ZP^V>wX7%-!;ilmrB>Y6*k(FY2?clFD@uGyEQc+ z-{a#AfR2Ku*})5GJ1WVI4<3AW#r8VQ-UvzCln^5|@R;mrK&0x*CR_uSv=+~Ked{JwvpV_$37~-)j96|L-i$EO- z9jPLo$ZVfEi>?Q{P0MDJ_PV1c)`>x9R9iIXT~U>GA3KZ2_?}!*c>EeQIJS8s#RSE- zRd0jLF(Fwvif|1`8KPq~oY^(IwePF1t7}@=VO=}<+qSEa0YQu2pFp*R@vFJ5Hnl_>pK*mo1%HQ&-&MZ%@e(j zz;M7zb#`Vl2hmeP;4t<>#F_>8uNfS)Wn-c|-{aSyszq))Lb#^;b7|WclZr^C^M9QV zXx=BK_S*RKc|5;n*;K0L?Hk%q-tV0(bnpFnub{2X>X1Fb*o6dEqUGaVe{?lwbwcX6sdW?|eYzyH?HNvt5Fw?>)Da zhlfbvCOt91uZqu5ed8AistS#l_i?Cn$jPcbRD1xKRSYFW%I%V6Hk zlTS0Obsl$My&JM2L!lH9X!nc&&1I_}YhrSl)~Ae}SIKqM&t|DABSi*vo>()$o2G2~ zway^0T+PIyjCItCD^W>}GzQVX;_IsxAUkB45D0wd6Ef{eS^%mH8pm!K&IS?&W4o!v zOyDk`Q18NT0bb;QWf|*J;x4OM4h_6^ggGKB*m+PxP(~^+QB~j) zQNFhW^J)#esI)&%r`!eApB3c(>dGWEPeyaQ%O0XjO8`i;&4r_UPL7sku~Y!rp%_|D z4`x29sxkC64GYF!Ke&Hj%g`1RW4)->H}xw9yR(%pk%JpRuotQB&ja1My3wjqdY3KC zJt%+4*Ib(_^2UmnYaKN#c&os%B4WHArXx~8K_29{OjrZtU&K^Rbx_mU>7v{w$`rtN z$M%~w_oQ$0^b~pxn}lBLoeWktZ7#cU3{bO%Xp$->n`*jTNFVA_!H>tELue++*VWE( z4fo&|`HkyRKK;05457ajDN7N_VI57K38`|0d+5K*9BwAKb&K$-j1&_*R!Wg}6(c-$ z#c#PyCzq=(ck(g7=GC>2wS0X6@oLz83&TC(Ko+ZUdoyRe?d%Ey`J*oeK9eH2c-i;2 zI~87YUSr3oJ7zt8rBRczW|7ipIL+Rqr|2s< z&EZ}zRil}jSxKok)njOl0u?kx@aHM0c;JM+^()N*MVKvZZA|2togt_1(d(HHgg2Fk z46XA=qB4BPOR^Z~GS9PjEa6fUj{t(|oUj9IgXI2@A>|EjGK{+${7#y-B|p#FRn;wy z2Yiy@ya%XdQsh}Q20PoEauphK@&s3w?@VNel_hwvcfLbgak!56MK?TSq>2`pH{Z6s zWX(h={N3|iQm!IdwM3&*Y~f&uhZ^hQ{R759=oD9>F7Mh2dceJyopQftdg~nkv=BUG za6Qc|#%kwEHj?WHv-Y0;C5zWV1eP{0>01l+?@DCT?q3Cd`}r zo3Yi)9=oW;L5#6kozVF4P;C_&!h^y*x#m;(g=-;(??O)zFJa>oGcC}BCtw_i3TKBC zc%??>u5a#>twkR^RI+%T6~HZjbd)&oUQnq-o0rO`fp?Td(6OiEa*Z|cb-fIzv7;(M zB_$;Ivr(Pkb9nQ5O1QLS**f%NI&QS5tDt%r^d*~itiVY2@p||Z{_$m1xyB?)_nDB< zPbq`T^tG{&;K(zLVuDgGROy@wMEx7h5HGrr+GCCf*Q z7oCP>?y8&5Cs#DN3~W6ZabWTaF=A&gv6*-ctMbUeHBk4)lE3Y(t-wL+W1P|ipWusP{PCWD9V4u4!1(!05WS)kaB&c3~M&e zISxtR;TyB&BLsoUaV2EB)%E4xLy0D~cOcWVh{3dOxw_$1?^u+ZSwe+a_2VZBDpDr> z{y{d|mls|YIo)L$C=>UH9sCJWsSU^P)q0mS|6HTC=g50X)t^o~9hh9`RC7n%%7c$uxM^@@cRm6l zsdnbh^jh4?TPC@xtjJze@Qn1|M@n}>r;8v4sw*38H^YoK;eWbLv6EKxd_EPNl-hT5 z<>4ZlCI!8?!LbDOr9Y(INucp1j<7yzmh!>BXcd+9>b5At{P;C)ILlk2TiI=J(5Yxa zi79*#5fT>)&4p&s(bw+eor^UKp))B$N#$ADHqw zV9kPc1lAuVr!o$|j&YrVc&OMw%e?7Pp6E&bd%N0VV)C5e*%;BO4d>^bod(z1q5Ep)Z_cqY)-N{$5km zBLirOYESernVLjs&ey4f6c1s=R!B)aN6EWlXB9q$O6Nou>)2#I?)S2|kLD&KODdmX z5pOIpYV#Xx!^X^TAnuJG%jzI&q+JtgV`eLgff_XjUo4K9bO6ghWJ9LPD@I@{eS|wBv@+P%Mqu%e3{B#e&4`%7S2<;1&;3rH0W@MM z-v&yaC`zga+pT{EIB$dh<$t`beOTgST})>y_Ss*TJf|HzHP+1ev^l0F%~$;N#dyA} z-YOvid>2s*{`yx;f~G3G@4w3p=TWwyhFb;d8a$6%)XP6u^6oKsNGw1`ABW5@H+V$= zOBxagstAs1)h5+s5TEXJI!ez}=C0J?15@39!k0c6EfbybBWCf&YBnvCnwc|35}ZD7 z+3Mt--Mb`E8up7a#$waht@Qxka?VMDd6Q+lFQ?8Cu|!R=>1WyHA6qm^0N1#%Rzli{ z#bX`dzYnUOTIlRIzFI9{H~#eKuI?SSNDKt3r}hGvfcfC}`VXfK^Nb9oRx4I?+1{tZ zGAYzA{edd=gX)G?Awox^<9{jEH>Ha;A_hpAF1fEmGu1W3Z1uhU^CUY;Nj?HynD<1J^HN|xloFyqQ1V7E-MD~Xs-46UyEfh3>ZMnx~RL%F|@~boV zDShao7k|6mB8w$!l^QkQ=)f??-E zb7Y~3M^T@^6NOAL>9W_v8Hz#QP16h$L$ZYq*-KlstNpMq#5Lja~kFImuH- ze;^CJ8%=7*3efs?fAZlHz^actn{(|a_W<9{km}hOcTVkD>qv(G^f*9r#=Qyr4jKX; z`sI5G=JV~1{psjjX)zO}R114##eYLvVFfAwZ?RSt**=bvq@<>rPS;zJ{~(^vGhgSn z6c=f(zdivt)Zdy^n3)vIbFk+lvz)X>>wdf9gyt;PW_CPBOQVd16(K;g#g6b1}RUpE6$7t z`Y|!+*%%-I;l(mj?@mN!n~A|5UPWqm28Zfrpfaor3^n0|1U?hM8k~lzunGBPx^*eZ zS|^vmNC{$v(BR_fD-uRK#^Vy5F{Dl_@~TTx{-E_*nMCmy7ak|BBHNn|@i0xE$k)#c zP3!Ectghl#MB-?o;QNywv+HCy!8mmp?W(k3)baE{4 zp_p8Lh%y6yEHjYNQ6pVeTF2l|3ZtgC@}*dD?WckN%bg#f&$MPZC7szJ=ox-sf$w0G zgL!FMS(jO5joUdfEbas4{gQ}p4;`w4mFl_g%jh`Cxtw{P2Tnx>*8@A{@X`3ve(~a4 z4uffkjkm+2U4gn4JRfxM3tvB$M^|GCPu`|~v^bPvQu|~IcYZ~G#(dA+=EsQH;0i1P zJFz%d3}Jl6Gl&#VS+J}=Mge|!$k5m@k}YReLSQ25r%!M_*$Z4^5qZXBe!UF9HpGlX z#J1#A{cCHh7JpR8kzkKGOlk861|04VcG1}^{1WBacnarToWmb=lU$tkMDcS*f9Vd$ zy40KVBOzJsNk)aJYacv1<*dt1el26(`eHT7>QCyqS3u%`iTw{!D5wYkR05k(o^GN& zT_CV7(sV|GuQamw0t5g1dNJsGfDM6dZwUbI3eelfJ%KJJTJ0CGIgaOS6R4Z{cY;wB z&JyBf2Ko-T0<7%FhgZB@vUG=o52d3jOgcEe17NW=Yt|czrA3d8wUYl+g8T)h@8qfm2q!Y^7zDkwz%Z?V_~+Vle@&2dmmBdrdo5lG9zbN6ZCAbk9|9OmDZv z=akao)Q%{*<1rU@6L14>WVAh%l;u*(7k9Me z9o)XlvdK3%t9|}5R&Az$#uf_X(>)~EJ!|Nj=>(#^l&la(zcUu&>MNUmnex5&d^gDp?p}s~qH~F> zeepZcMVJ(T+3zgr|6c|p?dT0aaw8T9-tts<{4cRt5ntEdMF1r70(1@T*w#loun};S z#`di3_6GO=2rya!5G}@oq^#|J;Dnb89i^+l0T^lj1{Uo^e1+XJm*Cm}4#3II3f(dX z4jBtw#UmTqE8PBx3)^Y^wbXTnV#5ee13;}_WW!d`cnD1gIKHelWOFSWfSCSm5G<1& zJ31yK#0}^c#&{M0lA?Bl)r5CQ7`CG-IVkW$Wo%zD@?wGQ6~@?J-~aXmJT^OD>;Y5J ze0JMwlkq)coIBr!F=D9yixyc0Zrlrfym5-L^#51tnE#W$f)+twscTQHVOH(pX%8+r zo(6(e3{v3Hj(1SqW?F$bAjFas$=3AdEy6jZjgn;T5Q_=yB@-Y65xw)I*U#G@g*L@F zwye*1@R<81jd5K=To_y8;lXgXrkhRp94t|2CuMjsqL+`zW~v4{)t|7`&Z;RVwhA*=G?S0!e1G2i#An8oxURF#`y;pJf_rLl)A8HUY;fB6KLKa&>Kv6 z5uF$*n6>^}r8?iz`uk*btGA1CD5V>4*I(SIV8ZfZjkNiX2^r>D^p$M40?>C53$sMr ztuS_n*dj&CBWl7by`IN7kF%9f{<=&$5vxG7P0vD7#$w0Z{I3l>$Gj3FUN8)@0n~^c zTfJ3ae_Ascfmv9+#vd;$Na=UO82@0sa>xoH>g3<-TL3}c@f2nS_0Wo*? zY-41JY>hVJlg`|7SSSQ!tG$2ULLZ?S7zPgeluo?EAuL~-3LQ|eqRn+%4o$!n|IDl# zF=YNkM!jtqidC%oSMSK{B1}SPh3@bk-%)nqK<%!<)B4^P($3SwOYR7{Vatc7>}7m9OYjPoj?#{|;2dNntw` z_ylPqGC#s8R6$hzFfu+{IuXkHgY>Ku zR-s`(;~~>%aS{a+p2EiA8SoLDYq8cH8%G&fv|p^=3|-wGOJ6iomv&`0(<+OyG>+!p z8cO`(&DklAQrD$`havS^i%RBh-ekM%XH>iFzvefSE4Qk@WC>vo0uG7?%`TP+!M1>E z*F>hjlkKC&R#Ucc$i(x=o(ZM#4H+s|>t<&#wi-BarlH#CYq0L7Hg|J0FWqSn$udxB zRhwUT*|z^O+y?SAa3u6lsbKV&A`+tXElZ`^YfHr|Pjkr`8rs}rF`%kuqlnuk1}907 zb!KY2M%{W2Z-wEB>hZ~pjcja=frM4MyTAu9!~+MdD&SKF9l5CE3WrJ>o&3GSP=daR z!(>o`w=q%37bVqbB7|vZCxm#m-!Z0lS*oi^Hd^qGE)3%Qt(LzL;zZpnQ%AV!{7=bs zcUCyse_bz>B)Rs;Wsu5c)jMKt<_TueiqctagkImIXvaSE>Y3s8kr`TBx)FvZj1vYM zWKzOBCW+es#V8H)(x%UYTjAC zmu-jLiSrLI03y?cBOXF(1^(Hj2_rsHrS@53J4oVqDOzaq0#qUS!Ve&AL(o!5k5zZG ztRjJYQHG~ft(Fpy(;M>Lm-@R^i(!;4uF{#|TrQ4uR!;{p0iE${C?G5$D6;avfDKCl=-mU*A&!z|8OXHVgT&#A4WO{eB43SI=6XXpC-GGJ@Txf= z>x}EnZT}r*+hS|8YExP7A$+3F{P>ed6cm(A(YXe#vQ)zUbqJljd9|m8#7s@mG<752 zmFdcKOjRDkQ&zqOF0OaE(ig?$S(=X&DKLo#=o{ohbBAf7Gbkyjr;}E5f|$biZ&&GF z^_)N3pEByK)No@`e~DFkQ#**9O{UYlJhLv>IJZLFNYV?sNb5IFk&GB$!{{$RQ*qC#}P<$Ucnaj4SI)lOr{+~H3swgQ3k zdvVB*^@V5HhUoYz;seh*hO)l&U=L*EX^Jn2w7-}k?>oBSqF^mpC1?75AN-$8*3%8D zx~JD*b;6An!F}rv@`VO8XS`E0ul4#>JJYb(R!-d{04@xo#%vfL$^XgIuVOna{WIPS!6C!)KnlRY=clw)>F6O7D^d){d2s3-mj=7Et%IYZ~IHGQZLp`z|MgX?hKR#lP5U zR^+eh-xPp@&vY#ub$Vq{t|PR>Y&m2GMMB_-O%1H(r$zsX z1yEe?vwP}3Qg1JD7pLEbtWvAH>{y$nTc|T zN@V-=4Z(9ed{8lv+4ZUES@sbdUMW@ohGQWQ!h(NA(2fLN-Q=bu>s!WZs6+U`U+b%G z{)HE1WQcmgoe~?R3VPg(HH8y3YuyVuo+oyZx1L=s@DY->6kDQOjktGlk>iHxnGxCN zmX53sZ6Vyovx*_4v($iw#hQR|>e5iP6=9qRM6eq8;@HOMH?r(z#Z>m&__R^>jo&}J zKzYnaz`fa70v<3JkCjiOZ6okkt^^XL|8n`xkSdXhm9|-n8w`G4oIBa=U?<{KcdYdC z?EIb7(T%b*bP#O}O!dL#s?@M``b#R5U3MG9#$;-6fMuj6t0OnJZAhP6n&ZI1T`@#- z16Al2aHQB$77DjN>GVbS%QDL=6WfE+sU@dWm`A z3$9D4Aqv->OBx$x!g8q6oaz{`>}b#ZC!n!v`i-Uf!`9mV(+66+P-oF}d8%Y~Nl!ig zXS{r_wOWZ3$1tuv4cTM3-AQ=*(|0BHWV)7Ad*tofkBzSlwin4%o7()^%eEWIH419D z0HX9`xw;GHyw)$eUQ(&P+gA^4HCdiL)~`l(A~a`B)m#mvPJPV~upA;$Coe}?K|~d7 zcK9#fL&y#X*oCz&Hpz*)#dq7x3UixVz~=?*%=d8}_w@-Q-)c%JqH(RQhGdT)w}W)M zdfTkeUiQ{|GrUg~%nqawOH4+KdP~D9U8thap+SpH+f@$1!0sR0<6{D~dA*YJLfbic zC-$Fx@%hcEcP-cQQh{m_g>d*g5oZI7moI>V5jRSXt*GBS%k}WM0l%B}Q&d%_3KiXS zou<-*^yI&raaA#4uq9)Wl`B1AwhEgnON3;?%{x+cS5^J>^UX2pg&)NiBgMj*TpM*r zn}dg3{GJ!wq@;}UI)MV0s$TgEs^YcsL&BNhDnDastTEP?B64w`Y$wWEXF~lRD4%_` zEu=%?cyW|3D(kuYyU-`6p+O-8S;!l7%a5AzU&*k=PAx)PseH@a$c&SH3vrPzc?$#e z#b^0AIk8AcmN?W~6g92}GEXSDr9zVMeB3JAEG){!$IdKN(N-uY7;5&*gLaS)WUv_Y z*+38g_`U~mDMYI8GouApQ~rno$M3v6ajcQ$vh75jS2_K9>tR3`@X8w3tc)Mp{3cPj zvGL`2U9sab!Mb}TT|GFRX(4-3 zWIkXGE|sxyfTe`!cynxYh#vo2l^ASHP6@A_BuyTDaVx$l_!~3}B)71(84bP+N4LBR z3A9BBVr=PWJCeswzf$&#=Wm1xg+d}IVYvq?_DdLhF35FdWq5n#{qfRy8Q?@N`$#&4 zt2)ov^6#!rG7mT-b4_5mCLT58q4yr}NhizLliYf8*7lb)UgZ}AqNTcZzoFu%i6BbC z-B7elWfALv63CbRC0f|O#yz?YoJc7wWb}cEFL}azA69VAef_lWhl~QaX(GC8y&Gv% zRo!QZF6_qNJJ~VzhZar~O9&?3AjA5@xArj)kb1jQoM3k) zO>fTIFu|Q#O!=7>Hln&@p&@$q)jxDr40MS)^S6y>^l_w>*+xD$%SDb3SzRd~(k4;K z5T7F`9fndQc2+c!F9`prxR%u;Td9mv^V{KdFTvFVOba3S6oiUKUBz6_)qMj6lUE*t z_@F@=$~gTXXl7i0vU}SUU6ArSg-~ksi7%@E`EcuxhZ8LiVT&cPOY5_4 zEAH1!dmAm>qURc}TXs&yfdJ1G|1XXscDc)sS`J+&xe!og{bunz4Ll_kW9`l}X4hpY z_q>ZM=IT_lKk07V2edxol1`PfOa$?ndK^Z_{ildk?YHYamuI;-EC^j?hwUOn9_x&Z z)-_xVLy6A!!-`Ds^K0&uPpC99n5wQ)_RS9=vz|TOg+d?@zLSWuDvSIpo~_>Z6XqSN zK-e-F=#t|dK0Kl({;$yWFctri+t)dw1skDFh1MOSX0R>-_8TIHr9{8gzU}9g)wS3D zh@7=mWdWF!DgL9)2%0s-))FaCe7%J^{YugA5pOc(hH%4Rp`~)#OHR{ZB7M^CCmR^= zP@S~e()rutSR+Eky0;LyMQvPsay4Fw8BmlwZ>wDnLyZ#?{p7JO!HP$)^J8LDb*h$@ zp(quJ>ISV*Ka@-l3MwXvOVByhPa2+0197eOe`SKZvL`9M%;Ka&tSasjb<90MY5nHQ zcj`StL%kqGN@N|LuYGNl4XQ?kF%Ot>CS>hDjg6VnAH_r=ZpF%Q?yJbFVCj$x@c3w& zcIZGz9-Y`tQBnvaXGWDh3sno)@FKL(XPFj$Pl{{?{N&zW(%b6QVzF`y!W7=~xgOeO zktHAtaQ#vB$jIX@PM^I6n)7Go%!23EbH1D3?5V$ri<B=c?KjV>nm zTf^L8Lu94xTtje$uH;f6AVM@LpXXVue{?9J{TFK&AebX#T}99K4Lp`E>{#g-V$YH- zYPWJf_^xj3LvX*38Wl)WvS0x>z-*MM{4`k7nWcruj5=%+*3t5$uV#mwb#aswrPj%m zB0h?SH{f_N#%2{5r4FMcMfXO%u@G%PRXx;%Gk)5&0e!1BkJk2drSx-mRK6o6N^$B4 zbu*DQB)ff?c_5-yXO{F)>X4lSx<-cb&M7aG1v?Zd+V$z=DdEIxX{?cR(XzAAaz~pY zBM+;o=A`2(yLSui-FU0g3S!L z0dsezJ6P;&0QqxQ{I;k1q<12Ao2l=!x!H@gv^jS;Ack(v88|sPDJnILEeB5qe|xO| zEP3Q*z(0Zt*z_*YbWodO&waJd6>EzW8$Fzy(RdKw^zqUWl|-2|r2KnLr}q!swv`PJ zXI`QO?i*;*{8ZJm9KQDA5Rm+wP*m-=&;O}Ik!Nr<^fj5-J$i4ZTPjSY*2}$~3t3%P zKQ@rPuI(6HN>-qHelGVNsKx56-_+%Rh;bq_dX)Q6 z0lz62Ojkr|hv#GDPrVJ#2$mUx(JjR2q znCiO%uToK8ZcFpL!xwd&+3W>>R1LaRBOU-vHlLXHH zZgo$$&&~ZhnwziVR8v)xsd( z1ljD6p*sAxw0`6KY~0`vgZ~hgJ#2Qu_hs1mD%;nee}=u&t5_-^t|p`j2Er-m>QqU* zsdOH3crxRx2xFJ<1cp5b^+-~-{~!tq1( z$mHF0L{T);quG(VzK*+XJ@MC19acM4IRcqN4n_ybb8#=(-&6PlquCmJv-~mxyEti`2^Y^E%X~iptc+GwbZ6Z ziZ^({RthX_wrTmNo;rK}uXm=eE5zgW^wZ9#tRT>_eZa0GD%>ezl@NFGNga#s)eK0w z@Zs(b3l)Jm?$1nF_9LLicPbYHCWa@p=-1&95a^u9qCNJ1$gz-jf-we>bDMY#x$V8<4v0 zdn5xup40hvhCRXys0nzy(tDa)wDHK$u{j89Nk#1!?OB6^V@8K)Yti0I+t<#8-H}*} zYrM3yIXvf{Nm>0d&8R*YD!0=@yNDH~6;_n{3~{+<^a^F+k43YWu~dDU|K=7?3HI?v z91i!>VJXm$x{-57jn&(+@tj=H-HB5{*F1cDlZ>PTvhfDi=Ocl9nzi2t?U>TvUu4=X zLeRLV0Xj1A`Pg2Lzu~R_9apQgU^;DlTVyZRM>L#4y0d#%`Og_w4De<~ZM)F6Aw-~d zH|szZMWQz zM=a3dk97c90`8IHLpPhO<0gQ(oIJgYrw+fkmA(6G-R}jXW8j?bWT1=4rtA&6>VMp1 zJkwq&0rau^Id+nv6zzOo9@?8)5a>HF^u2E%0u#IU=hc7S3>5ZZZ{9(m8`=O_b{{w+ z@_(BJ{ntrQxKt6D`yPm<7zPrQ+WCQs%SLAtfRGo>Y1Olco;T}HxgkVfb{W519QXNmJgt$T_V*>0wm0z zxyxGej*K;|9PUe-TK_FyWV?{Z?!d0DfyhpoX|1QPJlZ8$BjgUSQdT#YKv*0EIc{pf zQnFqV@)$cGv)7t9W7&Q;P-^gB9kG;-f22G61Uv*QFlxJ$ApQ9Cq1>_qqm_~M_8DYs zFPdhd{>sP1v3e@yPIZqHOFyOWk<|QfgHGF(UaA3TYemgR}xVj(Wc& zWdrr@@cL_mNFc%fl4X_YaQ;}Yc9Vwriy*({AtR0JB$N12LI|Il`y#!FG)Q^PTT8fX z;Py1BrqIQ_skos7YDlZUC{wur1CYZycX8`j`J<&S4Y^NiCHl;VvG*Q79)rlDgu9?c zBh!gGqUj`4quj-eZLM9#ALnF}2s`-@JOb^fySX3#nR`TRd>l5-qF)giUu}FPxmzsF z{D9xwLrcS+U+|ECtKj|sw{G?hHlD$|k}Ksss(s@jtv38>79bib_{0d=Q1&p4npI9^00mfzOKwPHBsfoxHum59;Bvhyr9!+$YKq%U zCxy{{oX8;OL0)1hO`oWV-qVD~)_^iF>E~b%tv?9)H6*+=qDz&lW7}2G?eOSo(N(ljFKy zKZqhuyD(5b{k&?h6iV!nL>qIFWu!Uf)6-GuS-FQ@{PP04q12d?-g~fxLfW}7tIiW$ zrpmwJDkfcrbjvt(AJ5h954UX?fIXTaqS{|N&5t_CJN_iPqGdW{bK3nUVs5GI)N2GdPdsPvQxD@Q0#Q%>)SWe0x)oDL9g5W_{=K{A3>Clk6L*e1qVdK z2?2)J-P*2{S~7jUhJDoX#{O^r2g$8%>7dKxV8HF9 zf*8RLJ0;xUIBW-y;5Fb&3O@1J%A8;6OaZDzolH}AD(+440)oBIPQVly$ znukaF{=fo*_BU@Yy2x}?1OhrRV|vpMGDJ|Tl&g=N(a^|8=i0v`I;>PTh@-JvH?Teh?!}*jT&gY6qPN+OHxeNW zqf86%w?~8;LvCi>gB|O?61bIJgcI%Htm-&-8b*FPnZ{%}aNQd>Kh5!h34bWLtHQBk zinUJbgoIdrWZ6xV8Z$$WdAEdO`%8d0XOKXR4G>d(leJ@WUMhwS?8z6*RYtjwh`q0k zxjX5kf)Nxlc8;%Xs&w$2W3NAY$c{s?V!K8AK`usIrag*f3|Qo)_}`5Sbla~@!1BMP<~p=RC+a-g`BAz{E}4G;^v}s zEzy-6)48Migk?nn3dr0tySf{D21v}e_PSvoc1g8tOJTz+&#|an;C=PiQJ0*82*47L zLmy_Y#Gc5&j>uZ6A1H~U-2XTQ?|$Cd;zLK3C3QY8``oC#64{dC6-t*O6!PYsylC2z zS=4%}v8H4Hg~t_G8dgxA>*m~9Cisli0YpZvI6M<&e6NIMhb=d+0ZZsXkMu<6drUAN zSkap~-yD|`?0nH7$;jCkgGU)xTX&jVGq~eB+UQzc`0niaz*>K|hJ5!)n-}H3BcD^8 zvgT~RM#Q=Ka>uvxX_(|{=z2`blP%(<4I^LsKzjgTw{hWvN}VnL_`Qp^h{92P;V{(b zm$Poo-$RzIun7gjAfudN!%AbrOO*$0V||?6x(3&n&3?=$r+57{a7PcLN>>i4Ybc_q zLlwmQ)r_grZ0Z_g9ywpWh;?ru2fZwt=CsR}_0SsktG~Ng4_G-A3_5`)7~W|>Mhh{N z&ABhM7&{l96&1uc?Z6RF^{=RRmx{hD^Vg`j7kpR_D1NKwPWrMbkTqx7tR7dKGE_NN zj1yVmB-3K`;)|6n&n^U7Sw5cuXQjMbALtuc*Iw+j1o$5Ktd%iDmQGpr z5Gq=Tw?Q2HvoTV;7;|@@LWpvBGby`2Eg^sX@1#YME5;7ximRUkV9KS#6z`70? z!z%pUYSl*7PL04qqE>$#rDy}28ZRo^r5Ld>DRB3T6E8=g3r4I+c4)rg#gjhMFD<-& zd__hsE@UpZ%9_0GUXEISY&d(HZ3a#UvshsSGwOdrB~NEUv?kr%p2kIG0|7a}XW~0U zBihoYn8h0CHybZACRxU`Oa81-b5+j+2+q4;iQC#6N@8o45S>QRyL zF%E}L+#ypQ805SB7r_n|ZKom{HJH(3_Vti~~E?Hnip3Q+*&tWl^-5Mq33`u%-jt~A@+vUcuvmsRQ^TVZFQI`o&X z#)`Hkv@qdEJ9_q+J@exKL?v9|enSYox3x9ue`=LH=(Y`|=l=FA2;H0%+F0D)@}T!@ zZ*u$(?3aIRB(m)`*>+=mn{G=7eq^~0{}*d0X~!B$d63Tt&pSI@K?Z70Xl-v%8_0N^ z6dfoX_b;XsU>%Jk0IpED)bv{~$zcwN3rybHqIu8(3+}pK;tqvD%natfL=PFySl8UG z+rcO~1}M$Bp<%8fpk_)l5PoR3f2Ow1~8Z7?W@mC6d&Tn7I$94)b_%aDOmkIG}^5cEEFz0?_-f(AK!`g~* z>C`n6wZH2h%loj7vtc!cc6AlDtv7m89lOg+i{DM0;yDo8h(QOVCe=E(%+I69Uz9SJ z2BrLc1WC>8#i+xx~W?200LM@_n{@Rjhke+5n#YWB{-2fog{@ znjpB)efb@#$Bo9jxObs{_|Z#c2370MieDSa`-cTU>4~dQP`c?FKS>(h_z_O@d#A>2 zypHHl+Zi|6BhH1z>>vvmKobJh+c*9+nL+&@F_r}-Hk2O25KTAeor*<^k}=Z=Q0_v7V@reoY}?^OCnfHbtFY9l~kHf1^R}z$nRk#iO*KqMi5d6XZ(Tz=zIAzbS(!RweE-=|;&Dap#g9%}Z z9ja-$m{R6)4jnyID5%L}@(+fPH;F(&zGG|%JiK34L=$-7AP0vIl+>JZAexSk&#SO^qXiU5%JjvY_z}BQZbe(9e=m}u*Q+`)Tf{)#5JY_Icw!nArUEfjhW##}I8pfBQBHXmVwE z+OcLC&f0#sT@(HIWsKBQf#|3>yZM{pG`E4K-x7xrC$Ai5*ba+($0WuF*Vc%kiFIdH z$z!_sqH==wohWW?|3J-9xbiS3`fkg>L2=o9uI_QapKFzmJ~oWdJ0Bq^7K6cbcK~Mn zBkndx57Oe6cxUF4WlM8*aVA~rYLUM2&4Fp0F<-!G0%qjXpD!PeY)R@n5;~!R)tUH2 z@9m;+=D;tVEn((ndhOR~Jg^TsYR}*m+s*x`>I0sk{_(o-$YP+(oyc^PPtJw77kBvP z+)y9Ys+IiJo3QfXU<*A}Q7YOLZgbby3}%p0=j+k!;owf!&ZW45*IvlW=e_rzt-k#| zGCcLy03>T!4^hksr%l&5Ft36;O>se*e$TY1*oQYo9b8iDD`t7e-&I!JLPN?*7}+N5_Yy{<&8wFtt4cH*Z$ucQE^i-? ztKm@QeRLoFuq&oBcR{hr zy3Zxi!MF9pjfuJ2fJ|bLw2R;8d}2@8a^U5ir`tylyhS9_<$3Eb(4#|vOm8SioAWXV zbVNJ1N4oKtyFR1rNSpP!Iy!B!IuBK63UXaK=;?(<{s8e_4w!0J7+K}bob~+jY<2N& z9;`h2hlhNfQ3xmGuy>3p9GX5a&Le(nvud0ULHqhTKjaR*Z8H9fyPf6`v1fYGuDR#J z3wfAI%WBr;ql0JaBmrakwm`CZI9Q=13;rUrcKhfnFNj+A`n!y(!;7Z-sFZ>;ErR@kozip3Eg1k_4$}5H`lG*@o1&~tF@gax_0SzRLCTf|MBDP5 zkarP19~Jq2IV9czG~%9^<%3K+@u{wI$O4;<`_>JJ6q`9hTH2K6n-kK}Pv9o))stb? zx;r>^Ys1)6H$+FJXHei-9ziCLT-e=N*dMvjwV_5EGN`LuZKpoO zr-QV@;*&gy_%y3rvgptYf6t#{;F+n6TaxfUQ!%lxy4R$eR?FN<*-kK$dn8z$pSulXZbtdc=Vj&P`f$#q z6ZYsJYh6YtbCo_d&Z^yNd#3SzhRoM@8e~X$9%U`oewDT2?xQp}{J>Fy!+1&|9WIbZ~+}EscG*IUKuZ(7;($AzyjDPaUg5-Eyl) zaL*IQbswN4lV~RzuNSBoA1-23J#nc%K%D*02eT-su)AKl(Xo1;Uu7klO->!8hbvOT zVCdP^@VCp~E~5PXaDMcm6s>|%k$6~@j{|<%w+dasb)h^_JwmM%Ru|#v1bkgd{ffS} ztCZWw!|hRS%2z`jV}w?^*KBAo%CcJ`q1au36J5{u zf#g@~XO?9D;Q0a7@|u^ITJ|`Z)4UI7s{m)Z))$BaoWM_aXD{)OQc15ZNTus8uAB8k z`Kh4XhNN^tGH#I&!3@mBfCH*dR#X)8yVb3*1SYyrc9z$mSL9q5oA?tB<> z(|NJM@GM7)s1ZKRei>V)#~4Z^T$vMNNXA1zdIRQutJk*MyxKiR3+`$D&`w*71|yTD znoUaAZ|Q`pWT!jR=s!M7={HD!Jw@OC)^^|E93l6&Y%+`dXCudT#E;2*)7!x#=+d?@R|VHUu2trW*O0iIcz z;MR^SZr-=Qvi}TkORj3cBznDfkJI&cbSXT!Xk-1{Ge@*dk5@&>OltVJBi|gK+`zC4 z&r`ga?fMQMcu2^@`YRw+0A>oY*(Tr5Yy>~8a5k4EPkM2D6;}4Dyc3B2$(CSev%Exu z^wlfSkRTC_?2&g>4QC&u<{y~OUyeew8oHyVJD2mb9>_{_^*=IU2QS1RW8Y)@8t+}R z<^CaLpKdtMd7!%AvkIXZ?8q#0Pago6ul8&mT9~i=IP|M0A!@$%%uSFqo}KSZ`_kM# zY>;~;@yGU0uMa()YNZDbjZMue`T8TbB9BhuUFuc~?L|aHD8VA5FM}@s2&rz7|67>0 z^c`U;QVOe#hq7y8#Hzpazv)pBKcH}|BU(tiPP#UlDLvz|v=Q=i%?d2Z#|rQM(Cf}% zB6ob!SngT~Ux4?C<|(NU@T{ZFY+wt_RCc+)JRQ>Io4)^t>LO{nm(hO6Q(b|&-_zRm zC5ZP$NPR#io~8!%J>lbSgz@1UIzxk@gCv0Yoi#8rbeItiKt;`LIbmPg8I421%BQ3J8WwzkR%ZE6C!bX3F% z|7ygsA_8;M2U6k|*d^d5f41xB-jTGKb!XO7l*WMUdc8aaKU5?1rPUG!X*wMyn;*{i zf}Io5UcSh)TrBZ!FMJQ2F(5e76QgD~{RK-On?%V!idAz8fbjXwS1=~SCaretX zapp)Gk4>&OTAmRDSQ+(P&YQ+l{-&_|r*F}q(&Dj4-$iO&Suo_CWIDYx(&PJ@{RH!< zR{PIz@06wQbxga~*<_O!4N4@%(LckLCHpX};~ zr^CkAwE>SEZxKHtpSB5*Qu!SUInZw5iEI^ga&%-|7E9P7`H`ViL(ZxC1F%#FU&^5- zV&^JK1gSQJA+BdPRmz+qMf5)2yj*V+d-VGlh}u(1BKeWM=bO2r{HaWEoFVJQm%Fgf z<_0UMX2y}`wZ2$x|16B^;7Jpb+tPK2o5eJ zq<-}DEaEADy}Ia0VDW*&{A6erke0_d&s2jj+&`60)J%6Fm@tZHE4R8I;A3xKIY)JG zpj$;XX{SlmUe5a{vhopCgxK4=lbP3m`!d5i28pN3yj&S-d5(?hrHFWz5Hnq>GF^x> zaol{ZTPnUEr$+K8`BRSpXOj&_A(RJA#%?yRq>6}o8W?}`)?7I|D)u~pwRdyIbZ+_f zU7JmlZhXX-#dp&9lSSRUYlzMQ$V7BoFCZCGczB_#;nGO>27mX2G2mx(i~3z&e(99% zH}%kbr05op_NhN*ht|g~kbX^|p$P^GI$V4YTPUO=R6J0umXgLv-G@*wYypPjK!$Hu z0c)c$!FPhh|FP}Igy;N_U>fJ;$Yjj56;94IhJ75P%45VGVL6V?9n(zA+JR`*t115h z(ehyYTEn;>Le&U#32RlR;e9XiqN|Aijz+9Lh*Q& z0M>Jo?M#2ALeq&9+f#RCmPI@RDQw<#SfZ@o?iFvTM$ z%|T4CL}luFb|MV>Gr)Xt)h!h6yuBX~R~}0w*nru-HFessZzZ(>*14S2p!I9hK=53Q z9}@ZtYdY6#KF(MK?bv-aLit0Wqa3h4j){!(+ZSp zm#w)UfNMb1xTa8<6yf{^AL21oyYhhVdY2wTK_XcpbM_vg3lq_%@|o;Cch3w-wt^|2 z8AF{2PuK0FzN)qU@$(-X7He_yP3=G&S4TXz>2n63P7lVvUX5HBX!(FAUdSnqT1?N)G zeIJLfXZ!lX>CpH`C#oMm$a>Qp0+5Q{X^v^}YxIZR+yHb?G?DKy7k@~H0^uQf-=s?C zkO7pGrB%x@;Zn_W1o-|SR^8?ZbB>s8Xn(3G-Z^SbQ@ia&;o?%C0>bu+K|0#<6q zgrJlI3#ab$4Fs+*{n4rQst%Lv%Rh(vF}5uLwxt+YwMx*xIi^E4@2m6jJ z>De^!i?*`0ly?iAuQWXF$aESJL5?})&pZ-8m+gSJ0nFmns#sNZTr{j3!H>?yvweJ$ zj_xqGWtC(Tr@x5Yb)9g+k*VlCPG?HY>Py^%n|k71ji3ODH(^@8gA67FpK$;w0)sg( zBMjX$Ov6-N$5tOO)#7|+eemZ;ElL|8OAMUP^dIaN_Fr(}0>l;+oVAOu(vv??0e%&B z=`^q*#9f)ZFsrmq!+Nb}u-y%*B>75P+R&j^y%yDId!LpJfq*Q@W*jlVz^){Ir9DJ!mp#b)dp3<5I{Rf?K8 zJ-m|Iv$GuO2p7LG7ekI3MFjTuGoc@Xqt=n3I~Z_7zLw8zQ-TChIt^9fBwMw418!J~ zZrgCSIUe_yGr`D8H=EV9rQFw3?^S-B-jfZ~cKI|+>21wyLK zVQ#Qa4NO=_UAg%}<*Q=GVx?&r_Er-V`is39fFI)yB;M~*AbifL&Bw%O@7iQKDL+j!3 zK22Ail8jfUeB4+4L`%YO)F@7SU@W8T(zr`jegulq7y!$9hOS82(r$8dlZHyt;Zrf& z^8^fRVob3O@gw4@w1+La;@Y;%6@xT-(IK<-D)|6nrenC?zrnz?NjWt6U_BcO7WEf# zR8KksOXd;s(T%auuN4a@caO*PZyA zCBKG&6qS9>iK|kB&Sp)Pj}FyDJ*`U163m_+;h5*=Mn}@bLC#W|=J*5q6;uw4sJ~A2 z@cjqDZJ3@y>E65N@sKwxJ39))B)Y=ETTr=vQ(JWZ?bvs5r3QPPD8-iJdlb$utOzOI z1xe%BnF9TP`twlrZW;Zn)uZ8%mcWnV-b&iof5nc3zrM-sk5@Lo+JFuq8fEbts+WBS zz}`ns8L=!WK@YW^DfjTofo=M2FZrIRFmRjhtrHTi9P)F5BZe`(L;t#61fc23-cO;H zri~Ja4e89r#-UA=O!=;P?oAW*gi)O~gUnN0uBK%?k*cbT4$e*cv27e+K5+wDWn!7p zZffA@?_Z0bO{+Q_Sf`0(meKr69i2~B^s~#1PQrQZ5vNn->_E)wvpWUyAqC~-MoqVC zgA=?dT*Jt3p~P~3AeYDUZgXl^Ca}=U;g;u*QVhmufNE9&d~SYjIHUrisi!K{NLk)U z{r#&u^YVi%0JDWaDlEg@1)_~!VGPB~tW)KNS9H+7d%f;;Ag+DSt4PCaB6K>Os;8sl z!`u9JvxQy|=_gtEwMf|?xLyaG%RuNfr0@%gv)TP-cU3Y@JhE-D!kYe;qeOyMK$cw= zr~_NawF75<^m9b{j$p0=uy1!LtiMDa!lXo>*c6%oSVqtI-FO`I@7Not=^%hhy!eN0 zv=ZW*L->+4a<86~5RDwC{QLLyJ{9q~`*GJZT)Sl80pn@^ehuuKvOdd=er>+Za?tYQ z<6Zunl>~n8KY#uI*nd5oW$sQPlfNm?yuCZ&OQ49zY{B-1{9h~$NLy%EVqv=dk^27$ zxUK?ZDGg@zKb>cU8TP*F?oqtDCW2&4l>i|d?+%KZxZpv|{AHPSBl zH!!@B4Rr2?#op@iPt@SQMqq-G7;sZBP<#{MH**27H0kKfUkvBfbJ_1%dVH^|0Aw_> zu_z*LUMBd9e(Li6W4^NkW&lh96g>lQ+JJF)7MPldkY+BuMv$t5a&Z_;` zz+DNg#O^@5GV7M&*2iCV-%5|zLhyKw*(e~>Cxi*q=54t8^`%N_OgzSA`-RS1onk@LCd;w*)?p+d^jleDAUfl*(zoH;oSluA}AhmD3!O^3Gzp z+GmaQ&y$kdNaX~Bxs^^loobehFAh~BH%bRc3)xp?8_v`xfT%LF^eUuTc7RpGoZ6UX z`ng*D@Oe?ss;zQSv6;%d4UqMexkCC&rCn-lWooD8FTI=nK7Xclk_|NNIfjG-GW`5b z?f24ptl?Ir*~%5u#i&fW!fGo;T|5QM^gV7S37@rAURIyG5p_hO)tYhIojVTg#( zerf>3=OGj>1AKp2*B+^J1~_<9RG*~J7Z7pcG}3=WQxY57*#ePeooXJIf?9ZEbtrRV z{mk`xp%W%%aujC$9w$P{eSm~GR@{nEZYnn^uPiy*-bQOlhu`-8RT#GIVR_i$hm)}Y;*fR*G z>0u*6C+p~+U3T?R{`Z{(fEr|XWmnHvur>~vs4D82WqkY1Aj7NSJ_MA@h|!O0@S~Q< zl~65ma1xy|Z#j$&BgBbAs{Dhy&q=_~R_t+d?Y-fe(m^z<-#3Z%vom1U+h@tf?0wLJs+Iit*WX$<)P(-;&5bg+&;AlVsAW9qpM4-y|zQ*I8 zFIXMVJr}mD4b*!8J|_X5E1~yY9(P7sP2~Ym)tx`$L_o2bA-uw6R{SVXi|R?Ns~*1- zLfEm%68AsM<;8I4_JoV!4Rn^jFd$LPaek-Hp^Hl46(ma3tmN`bK0Rg`Z>7`>*%+Ie zIsLD3M9l8fNTNHA*r(hjqDH24$i?~uE5z%_T5LFo**#_PIlyAT!wCMx0w{siIGlV; zLpvd(>b00!aS+(IcUX$y}j2 zyL=GJBg&Pm2U!;_nShXE^P(c2vFqffG*$>%=m}9yZn37MCN$p>o3S~NfNK{PS2qm2 zaI_;T({5LSV}zZg9(-j@5-kaU8eNDrN>y6u4(f)4G3__$a#%&FonPr@-L#|Cf8dPO zt{0tiNZdXEPAS9nvCo=WqbD=m6&~t+vZVZd^Xu6U=kiWWS-OM0fLb3-c0L1Ck|24)W=%UJan<=vZQbB z+6T}{N#L>=cJ#1f(M@FRWhMB^d#B9Pwho-m;@y)m0YCvr?i>3)IPrwA1y_K6WwEM& zqz!EGZL>rz#Rt1mPvGnh`FHIw$P*oJyT`32Rx_0giSMLhKSFog@G}v4Dbrq7ZsO1cVTJKp=nx;iwdqCLKg-BoQI> zBnk(R5(Nk33 zghc3+9SWA5FU-zG_uC8s*bo+DWG-A)syGMr)<9>bP(!PBVV8+{@)4LvrGH zQR!c|5swy*|GK?=>3{98P@`i^)=HsVboBIhf3tOqlT$qxq^ehMV&lhZ)D_`*S7gm)FQgBhX>*NiHid&-r83Rr4WDO)Ewo_Td7*WHb_S8Ufp14`zHyO;^55 znlV4O-{z;tye}WYRqP%`x;DN?#fd#bgZqj!eh!;%8-bS9Tbs4=vD*?9?0ok6HnWTV zOL&r&`$nO+XPC}h9hDr2fzg<6>}>Mle!CRAo^f`-&_{(VJFrKXb#jvdK>-y0yb&Ji zW(+m4vQa=zs2K^RK1V9Kq(r#++h*J;6S6_#O^OO921;|Cze8IM4JDJ|ATjB1Qhx-h z9sQs;=g>M`C5B*H)NMBZa!jB`5&wSOpGij6lfG9>W*H;>KG@D1m9??8z?B_h<0|56 zv;L|@9s$PT-)^RIX1+hPB_;O><60SR;y0Eoh!ZvL8$PMt$inxX65$sxEu9K*69Uu& zBC@X4p&nxqEhcC5(|!}rKq#Lauv9)Zbk#C=D{%H7mRcuec)62RK+jx zt?m}Voxs#6TM7Oh6KnD9x-M?yl!FAd9hqxibs~*aXY8F2D&TdhovT@!@1NnDysJXZ z+CHUNrffZ><37wRzc;Vw#r|w}om?ez0X?ff8jGB{f`R1WYqC5-mWfZlyj-;6s=DKc zUoK93X5DtH9U}SNMiTS=-F&;&{%M_vHn!g6yq7bUB`0X1@h2rS+O!nkeS5|v@96PM zX@lEx1WAc4v!{b@l8{}V47$eO0TX#3@kNsAp^%p^^#6Qu6}Z@#s-Y3a&*hkWbZ8(A zzK~0K+?U$-L3DhH!VY1kpolU9dqdVs_9#`&q(eG=Q7Z=5@+vL-K`y)17muGqCJ}Hc zeR)ND8i)VFB$Xk86bdF1t;RwO;1e--mYnwYH;v7n66|bzZq?XVQJzG&kTKM^ly=T8 z&Xg2^p6PW{ism@arc8EM`=bBZs5p`A3cp|>i4X-;uJi)eYWA^J`wSzmYCQw5ku>nK z*umT1(5c0YNUXB}yR0A=y+mKmqz)&Bt=^tt)b;tTOAqpDy|PPFpnY-*Fh}Oav=*o+ zK@RvDFta|BBlBXNh_RQyABT(b-DB5qO=D_2FXQZD{- zE{~M{>R3?0>%eyFc3bf`4d+W-sxCg=E%YXa#!5Y`fNZ&uE7MKe8M`0K3u*Spy3`lH zvp5-Z$+tXMd~a!-cAsA(xN1M7?1i8+?qan^S@@^a^4)pRRcU{#Nng<|_r3ksWQoF5 zh1-1MWzakN0~WNE)`|;pP9duUDDBFr9dVFfe$P8{w;0Y|IhbW!J@c`e+di$Lmh*Ou zL!CHX;y18<&WCc26iupXrrR4OOYN$7yXm30(2`0K_7mNgb|&RYN?)enndu0-2h)k? z5TZ68!#wb)+Ngxw(_*f)qY_thx&x|Z{ikNt23C*VMBP-Lem(Q-93qB-25uA+cz@= zl12)YhsUQs(68zp-${Aj-LV_fwgFO_Ee$M&TLg0)_tf)2O3pYqcXqLz>e!f|Ely$i z&^)c0Ee$wfS`(DZ48Au-T1gww>O<)dCRN)u_c6+NA>9WJ9^dc%%YT?smK8981@UyN z{+!*n%+C$r8wLW)A9T7q+7skrz=U}*2ue_?`;b@h<-+`9@$8{f1xxkg?fT z(9N~kp0hnJDGv2u&f%4RBGLA%x7XW1J$_7Z`9%b*xLv0K451I0)*d*GVD|p|81H(L z)cRnOglijitk~5}9$zsR@J_C+D&=f^lgBc~@C&$O6neOUuUraIqBeKQYsisjka4yx zS=?()XyxOs=EFEPl(<(4Ofv&M#Kdi!6yYMlBN;#0Xmc^V|U!6)jg?Seps{_&BE6_<(l_z3S`k(Sc5>01o z7GJD279F1gV_x>g$7VcQfZxAKA|Xk(J+tpRuLSv9FCX8U3l+D*Z&@~gosUZ=26fGw zvp+}}F-I+l&4bIm;9o|0njGna`Y(Gce5sh?VZ|DZC$7~RyXpn=o{ zlK}Vm*Tr=hLEa=ahsv-~9kY};**?%AZzA`=UD}54;R}rt6nZu;Wxt)sb7+&0;l%K? zp<8v4vuxT{5@|N=nw=0PT20u0P3i(x6ZTDitF1?gP#{}+H_Rp+Ei+x&fAiy+6xY55 z_|=p81= z16eqpHsFiaX*PvFywgMW9Xu2rd-+1!mRMTnxZx=Q&%|*|ow9lL<}7i#iuWT}KQ>Rx_P;tJc9^8)=s|FI;@fXb9u{&ykX?x~3>Ww`(N~^y78@Gy_tS9+B z`weK-cKmOK!^9WX>0$uv zJYtuALyF)gW%y4+zn!C=k$T(!2|zu!=ZKxZELomF`2ugwbn zD=60j9l%U)_LT$}vkohJpOaFh=`>bj$>wzW06@~+gK*)Wl0%TP0eB3- zWR`S$3?E}!%o<32)7V5AQ9%$~NypQ&`tlZU`w9O>#c-0%-QXX3@^%7yQLz}(BNl0xQ zr9Er0vJ>>P3|mS`68B2=}lDYy^#;#0|2Xa zR&Cl6g=plG=L&XK3bWbmyfK8+Px5@7x_)frJAl z0W!`sXJ_hx6tloqcm#8A^HI?FU z@aK^G`T-G?wOsQ%)RX&4CI8!gqgPf!x^RDsaII(PZWr97jt+(bPOgcWr6$Dea~BtNBzAb1gGPdaUM--uegLvSd!Un9S9 zPRhDyq(S&6?Noq+N5yD_ z&6X8fX|&Jq0wS~z_r@3jWV~;EzU$(C-Ub>sVpAwuN+?E3b1mF(uG;#@ubrICC|xkOwo$at5YOlu z4|tlHQw@O92;|9vB#g)2_dz8%tDd~uZR&;a1`=RDF*G9c!x76#OpPXVZzWz%dHPza zO|MrIH9+Qr+M%0eEqiiVw7=zk2Jp?zdnQO4lI`Sj`J};jl^m~N6;y1C6}P$ug-LgQ zx;#}+H1J;HB;ZX3YuevxN)#|TWyfVbDKaUh&@kuQcvI72r}j^S(^YcDMCpo5 znCA!Z|I9;4@V9fmg#6b}|9_AD`G1tFnx)|E`Sg!H??=3QNru7~f2mnHKnZ#M%cZ&h z|I*&^e>dR>!n`m<%hOmz`?;E$?+%_lTl1g6pVCUQ()^c1i*YtsF_J;ijG)%ieGtad zTz{wHqKfmmy!f^nvACIci=dND68_F%*N~05_^ps8O9Olp4jK zN%oG-Be+b%YbA2VsXNEFq17o-S)S+H&jx6#zqVeQ?nPtty4L8F-5179R-R}-T;>;ct*yFC{CMK9btm)fk*iJMgmSy; zb6-W`kWTlno|JSsZbRv^RxR)kuxay3@(1bI@hQH%VJEJ|9;RB*1-jl_jOmbF-I7>+k7EZmXQ|m-(vA>lAnfW_ODI zh}BkNxKm1yQ_79n93CLQ906klxbvHqmh6mMSi;9#4r<5oDEhJPv~ zBGNMwzGZfg*bw+jN&vtI#k7C)y?XLqH`~eCxQ()2Bj9nvEEuD0;?X?*UTz>vVg>4^ zh+LOVW0d3S@%kk3i}jxDHnIGb%4Nv%`Dn^Q|!Oy_7T|DN%l zQCiBKc{?`k9z6R;^tXKSJg)}5IORv-bIN5OQmjnwaXbV%Y=oNb+E(Uh8h@>xyQyKy zIk*2#(d}`&e+pr0|0wN3f_)6q$=4zrg@Veu-Z;Cw9jdYUUb>O)s?`oFe~ETBw(K#6 zIk)YEDHQZF&AAFnIWN!p5XRk4ByXKA6UpCHbVQhXlzJAD*@{MFo4d+KZ~p~cUFb%b zJme=nd$b&}z&vJ%zFB3s~M zMR^r{g?H!flmTaWDlOXWL(A#1JZFjju%DmH|9^EQ<^M94Ub}RFj2}P;!oW-vt>}gM z)+&}!JdCmH)n7^Ev$;@q^kqAx^uA7s}Ry9{#;y6y`o$HcbQNXvMG zJ8%f{GA?Fy$j6i#n6cRkv&-21CAqq~W(hsx_ibL9O98zvLFE!%ct(+|P=6tYdF>G7j;&(O~-%U7FAXH7|$hhQKC=Q5@94Ax6+6NiP^gz~R2;q(0!xj}ol zH~(5;j_wMb7nXy>lTlI{Yo>RmOiU?hrZb$}Dn-y0L#w{(BZ6H8X$RS_eL}9?gjOOq zKxuJZa>-up5X!=B;$u>xTEXH#vAOf=a$Q{a(}m+ZHDdUc3-qi*nqt1_Q}1y7gO)%K z!@xz~!my0Y+m8hT?Rt8xFiv7uieBFpn95Q*g_|T{L=|RFClxP8dD3S8+E>+XOiB#5 zCu-=5^QAKudPzE(F)XR(BNiD=PFvCHO@7~6F0EP9N7h60M3U*?$(H>by}zuz6G-N?bk;&I^~Yo)(5?e+ z>QlREhoaEQuI2c&?#DyT7jCxHbrhK0I#KPlx}p7$b9u{NZjW;VJNS+7I^wc81wG51 zurZ`YLm7+d)x5k7U^K<%z@J)?2{@&K!LFJEDWb>k(Ab;j^JSGhL~gCZ1i(#Z8k+AW zrH69>o|YU5te^&USSOQ3F707_h20X3BOdAxg2i!ll{v>JOn$t5uNQv8H= zy*=GFmyfcZ0y4rdR;ODX81S^UCe0d>maZCzjBe9FBlj(nBG>U(0mxm=?RUzi#yDvN z?6+`Rsn$0T5Vgkjm*mD&8wM{zYU zVFT@6zijw3r!UK!)d1O&9I!Gr8XEeyt&&_OmOK}dJxH`$y}O&3ALt8Z3j#-J%i-hG2kzu&z8x~zCl26mUaE@wo;p$!eK1*j)= zRw~t2Cn|8TaxfVa5JIoWt-jSkTU9-q_dx++kjwRU5D63xl{as%Q|}ywKFhx{ub+yR zCTUZrb}wo_On2s+-z;8?+F$wYXgye2KC8<nWMTk^IVxn)w;0sD_q2X%ynT z0PnGK@mh0*zu|-eE{fpLr@^hdvo(5u*`a^-3_I?k@Q*#NC*j(14iRZUv26GvDT z#DY8+l&VGPK=h`uy7mccPaU(fdRXeD*H&-|^63I@v(c?94hE1CwKCQ4Q8X%cBOR$0Mlr{}93-hliKnePnNRhv zLKEo&lMfHQYoDjJZj~|^8@7}AQ?$`8kka7#?6HJ7oN}a=O?!mTHQBBN<-R5 zp=GcJ6Ngb~JbdCgiIaXhlP9jSJ&b|g05QhO|3qAiKR}jezJStt3!?H!f(NAJ2s81_ zGkL?iVto8I8aF013%Jd!_Owlw+LLf@&e~cANm_A3vG=~4C4GpsnV=IG(*_>>8^io6Z5E5`LQm2Jg%6hQ z^h?uevQNRwiT!mV0aY8$Ev$L$fzO+JlLynTfmX!yh!z}fT)Nez@!>AT9A=9v7?Rv7 zd_}&kkZ2a>sh=`Wfrlh1gTYBl`ri%sMQhPA__us+ESTN|I29!ol)JQd#$mpqBKRIT zBP8z$yrOd>b}4RN@JyddsK1A>eMO~qF0xx%KkB&|fBIuG_>1;(WjL3VIPTiNliKX{ zvVqh<;*VyF0^VK8s4ASfcJYYPQxZY=U~3wuDEr<$y%aOXi}`0{VMN>0{z*czl;y6H zp@cB)SZRR_H}Clalkk})_{3H}dJ?X02X5dShg@(<$LnebJnYX<5(}U(EC}WEO5O+V zZzZXaGEK*@FR~OXXSD@;f`^vF(BG$mk0I7(=Qrl)I?WuNB|p)XflSa)2v2xeK(2O? z89Fa&%Nz@hn{9BgrHVB5%1{HX>VXxm3ecp4=;S~Bg4agx8QQql{9-Dexc__5!rQhv+Ga7k_Ef+~-D$)%KN(5) zTAwJsim`XS@6ceq_mk)2T75B*{oA)!I_#{HI=n@Fvf25q`hH`PaN}$ zC7`w#(+Rh(;BGRSs3O_96dlQ6A7#p)Za$8)mOC8)y2vL`s>vbo(;6$lDXDr*vSU{; z)PO2)D>1|9SIup637!ok)?_0-dxO`Bo7}cDew~IzvOgU<(!s4?glo%FY!KSE1!8{2 zIGr(z!yM8LCHT4ZgOE@$yeu!okL?n6xu;N=$JS_srGWU``hq)G}>H%9k3d!d4qU{4vV95w`R~=eD2=fx1Mfsjhk}J zat9WAmsk4&*8=i4k{bWMgZ`SZxYrWh)#o5TzR(M$d^~<>jcjYTr}@sBQ2p`~Vb6I& z9Tj2@U^KOTkj{jqe-4>E)v;+dL@r+%>)G|^R-8^?g4f2=+Z~%3n{0kFzE;U@O||Mt zblEqx8AkA2h_Z5MeAvxV=whk!mgb5ad>9{QDOWMXht&lju3II)m!Y#XK6Z+Gh6r1$ zaSm#He=O3{^{dX!e`3k6>{1lBrfC_qZ0~B4<6uQFtdF*OVx>Vu3FRw(A}v(53A1Ar zjsCqMNi9~Z%qK6$dhP5IiNfH-VxhCXZ7wn`sU%9fa5pmeWOWJMKt3@%&2RnUh?=r{ z1?msYRoPf+1V-nhxo-OjBj@QVSEc!Csw2;ne*BxXRRjQ2HWCNC=cZb+`)INFkmK7p zr2#BIPgE_0Ao-HH!eB3Pa^Af4sCOV3|RAesdj>YP&l>|w&p%GRiRdw=YuT9F&t(d zPb6<5gGonvC!LKf&1a8v&HIZ{As z1ozN%0B6LE1rE@mkzx&CJU_hlLTI=M3Eyc$p8b%TR~>@-(jOUC4_1T2Qyajvah)p@ z`-2os35Tcs>cg%lUy9uW;gIY?bb#a%*E(3deBXfvpIV7R+Y^amC=L!#Ir=$7iMfJB zd0uk`efM2+A>%6epA{rxgiUXF_gsWc?bfuB+u-Q+o4O)Ya_da3&s@JrPfyoEZ#Arc zNeHKE2mYeP0WwbxQVOWq?;Vv+HCrquT?k5K{B=mn+q$0gg7lBc$j4nXE>WWDqodFu znev<$g3n9Z9nK{zn6s_Oq$cSHx<+bYsHZh952^SsJ&FE6Ewufh)5b8AJsy115Z%DH zTWkG`RrqKBeP;y1NI#ad_Cb}hax#4 zO7bkQ`98>Y>Z;rreC0kxOAe2s=S!4&KP;(NeBM})D#(YGHm#cY1kRe+`vs73!8rs- zNx(B>VWHPHYAkN&1nF-50Z7TN&*Ap$g2~7;mcuNsi*Wc#AS6bZF&TD2cW4wj@Up&p z&2xUXcmoAVLx=jn+7jP6!6^RBlA#Kw-!Iop%@IKpW}ebAZ%i-J5eHC+p?x*Po9R7Ggse23y%*n?4|FWYlvWS~m>euCu4 z*tbt|XpY+38pT$J!9h)2>s|Bw&#~11E~={7R_i7$4@ET-2uE#OV0L*4vKf-{#Gb9k zlJUOlxyBk-i^TU<%a(nC+hwE!c|lht!7Po~%*qAK z=q9u4M_`Wofm{cJ?Uvx^?~JepR_O?BV=iY3i4Gkf)10QAS_x(h)nEKHxt`PY#jM&` zf(J?K>YAJjKJS{D8h^r~Z{h+k1mxxA2D(6BO{o7T*azp7tiPOzpho`=ixD(Pi@QFo8^x8 ze@C?g&+8Y-p$6&{gAO!+t(rxP2k}u}S*c}CssMiVyg!WLRe5*`WAA~Uwd2S-CkM2` z#yahmo5s1K1re@=J)n#3>q4(A7lSU-+B^!e7M1SgEy|nL199Q_d zou~qY=8n(9R`MFe9GYJn`cDd)z8AULB4YM#3^(<$!OqTHN87`{aYf~|9LXxofJ4l3 z*V4?6sTfW5(_38pXlBP+nee&%EZW%)@T;{-_cP;!Vv;8++TSGCXliUbt5T%l(9NoS zm^R0trB6P5&y}YivsCSIIyR`MJZMOQB*R<%#Kt_ti@vTrj9feKhx``56Gf6`u%vVt zY6y{oZ>Y;Q0U=peH7n zx}lFrR8aedfgF+OzEc8dX*`AV^rv=5 z1w_@%qe!ZC5wGoOtqfi-y=PGkiLlNZf@mOyb-sOn<)I`n9^kC+XnxMoky;-8j_ zSuP)xE}|E!YUUbVduMoU?~3F58qyG;`O8~xL4T?`=`_9bHwVAD+d-`MJwLJC0YF$3 zmZc%x&!(wCR6NH#+vbW;8)6(CBJnuDs1((xVj$i692KlrM@Z8g7F-jlB2wvZKsB7& zoHd=YqsOX?AH=PqvwnmS#E%g2xsO(ZsspwdOI-+)S<}@Gsh;Ny80es%W>U(=>BF^{ zQRpWVtGJ3FOGr{WqwEEdL8^{im0o^d6K{ZQAlU|eeX%#Ib^mj$Ct3z{Fm0_$V=6)f z=ol3iqiKAoasNsTH91Ut3X~rW=NV|3pFyOmJXf5bF>~d;|D9eW0msvdB_z6fmn>f@ zI})1r^zP+>=~u(Tl6^7X&g2=`&$@mcqlwgo<;;!9!U8Nk_h|ugy6CyNWDU^%#Hp2P z*Go5+FLxmCvOu9h-<2<1L)L@aEM1KeIy#jc9S)Zxn|No_C0=b)tbsH>A}#vfe%8qU zYn|1=w1|a7(t{4wI_-pqCCJmB4-W#QrTz?RbLqeH0uMJ2SX+Ho!D*e!=yGkCRQcR}3U^{ETKw%F4@HsdHJ@bSgY&Ryl}MK|hIuiq zLMKjUnq!mZH#c8{0|jRjLjhi%*hc4)XKT?zQ#1J?YE>7?Dr=TP=6HOcT^rH7}w)n)8z*MANJ9daAL8We=SVBqhEaA{ET<#G==}Or{P)aCz-t zU-~kJG~MH{03{0Q08^&oyG3y|RQrt9h2C(AI&2h~^^+>9ZSRFl0NxfTf(-)4d^#f1 z*Kb*eFA1}#Hn{#sj?n%Ot?@(3pi^~{bJVfUxwXP(W_m`4I&0T<^c|s5wFvPA*Rh;zqF}kshocXF5Q{$M1#ADemXm1UKf_ep$of zO2Zq#fb9r|RttbvOS^BKSHEnRftWk1JhE<|cbs&Vth7V9Y59+DJV5`7LHnXh^h=2A z-69j(_|YruH~WI`Nyt?+I~i2t!p>39=5s`kc%_Dr=FL4jc7I*Q9w43Fp`N0yJGMO` zhyG4h6o0~uU>lALM)3RKF4NI@A(Tt&l z2(?A~#r^Ik-SNNfcvQOM9`1B)K2WK{bg~SiO60y(n{^N+ee!DAn9cxmMmrG}N+dy+ z96Z;iy|kRjUJI3vK(XBA$mQ)~@}u|{3&&kg?YxNu(j)V(qcxuXD^dfU$ou2Cu*0}N z{Y5xaF_p{uCY-qMWSjtLzRTooBee8Mn8Z;wd*QJTix*m!i|{nz+w2}$TPT}*4HPHJ zMMb+f0Zsz|+0O)ZkVw@Is{Fu{)JOM0d?Js0F4wmXvD2R>v+nBJ?JeXXGNgJ3(5xXP zsl&K>0bpKDlpv>OZE&8xnQLXnxB%O7aqp9tJ9>$q-tc8o!L5$#fiP~Xwnkfy8ZfAL z#Lw9qZ4-Md-t4vXOydW8Vndb3M4qAPVCLqMY@SyWhV{@Si)G2EZ z#e)?aHJfw~9ILp}`5+lDpz=rG8M+0}hm-=BI5JBuO&AA^vqQaPA7r#;l#<*!T$M-o z2A%K$Y4HF>lW5#P%vh4Cw60oicy0qZ+ET*Nok4OTf}FFIStB%0tohEHzlt?|n4iTO zMc%#?!w0M;7|g0KCHj!GK!_k4x`?7#ZHK5SKlY$f_Ir&cd$elEyfjCfev243TU_Tc z7DtG%Ns3y3VcPAaHb33mz+oT0keCBb3*}D;N$cJuZ02tDG4D3I;97SDa?Tfc4dsn| zPvkR`XErLH_^vxT8EP-$ zB2%@&cD+s6Om*q=5X~%qox?nEMZk8F3O8BjgxWZDj z)_hib@OEwg51?f``2aa?sQB&8qRH;CN*A+peu@d2S*)k>wB7gZ`=I;xgiNmFQ5>tk z?rP@2XZbOkkXpfsOEyn;Gz)t%WhPd}Ot{{T+tm)eem}e8<+{E`Hg;Eo4;V~jLvuNqI^`7k%1)4L z8NGxLqOoeaRLa0+a?4vR=#Um#&=^oX4ZTRbu`3v2ZP0vaqPiG*J?kDVv3Y<})hQ9o z^r|1-R~8Xp^jeeA&G@hK#d6l(d37KOeNqu!GZcJZ7X=XYHcdKWw~}&r02z2k9>LWo z0-)H1n1A2YmTJ#P5Xb#y^#i_qhQXNslxwjJpG8mPsNi!qazmPS(N#jz%fhcOR}2@6 zrHEFp5pu2qk8yU^cgAkWNe zJbhJsNTngWC!=_zpm9{Xd9#z1+5by9O2~N3V7glCAfMK&K2Pw4hvm9^`2|=pr-QOr z%|nn%Vz?w)4mhO?ej1<4{io(d)20SOREd1zN)I585s$NoYs&$EHSe|-2AO7Di5e0O z?cs553|JL(?OfhubS8*P4ig=D znma`|hE^&+zd=eT_`1;(FDIpXe>1f+*FY_(Ku`ta4s;#OoBy<8$xrwoAjVl}M=y}< zMchyFIZE-OaXcsR7#YMwKidC$ELVTFw=7F_#ua-twILN6_|)&4 zAU}jZGdUQe&zzul`a%pz1!i%}J?W|halNGVPRn+v1JVSN5shgt6_t_3-!0c_o;5`$XsbY${bw_R7v8T7^-?Bs zQN^MFWjt>+aqLkZuVCvm=WU@gbyFeNJ8k{CvQ zpT*o;iv?JJcUGQ`;1#8wmB}Gv%gqwKw^duZTqaeImygeV0Uv&$l;n}J%Ufg=sUh-A z;r%1$Q-3V?f3hu0kCXrwvEv(Xa9Fu0o2#YIFM*DCy1&^;(h6jb`g#SKCC!==0y$P_ z_@F`Ue%u(GAteX4&5&K{wRHE_Dn4RQG@rl~1DpZ20&DZq zPv48US}PZ_PrS?`0!{9$6y8OCP+Q!S-FYzeN4%ce4G8{zN=nsjqJc1v5x)%99yM!c za5pVk)=|GG>K+G6E)@o$NkSF_cqzuQ&u_8cQ#Y!@9+iMy;wT{xaFZJE(Ix^r>9`)A zvcL}**Cf`DStG0CM4qNSSyHQi>)mFWR({j-7>55=Nm&FQeafi}zHJUII;-b&nR5lk z74BPPkFC;Y=)O|J5^QeZ3Px-ClFnZxrvg6nHSldKd^%9$iAQLkpMuQzxh>`XAoOCM zwS(_mIp{F>%-m(i*sPw;K1tx@mf+kGT%XUkdT?y$No#YW4*7oDs#nR_AxK`rC;h_o z7lJcT_cQ4}j~Y6P_g5BgMeeTz#`iV>IK&%a_EC6O?Fe2sqT`{)EPfsl@s6htnf6pc{%%m2R|x%4sgTLrm^2qu5s(P z=GGozb8Tlv@sm4;IKkZA7o*VPr$D~xA@qpGEWy>VJ zSO?Qy$rLd1d`zySdB1ehy;)M^%7A@>b1IWa3ruLTo&2%{z^3MQn&`-Loid>nVtdqf z+B}Foi@>ZT4NsRHWFBNIJe2*oAPP)Ky#=Td)D3}KPodv>uE?n4B~ppz+OI_A62(47 z!5~q(H?=RioU9DL#Hz6me|*o-%1L!OFxhoB>QTzw>@S%tx-9YYCaP^-Ww?CRQ13cn zH50dBzV2Jt4$A%A3jjLj)swUi_+QCu?m(Xo#I4rVQoJULnIoAm9uuq`JZzAZjVuAD zO!bb5oG1@(p2m{HY^cSpx96`tP|7i#e5O;OMTcamufj4CoTv~`h*x|G1&^tFhSmtB zDfBUv$1i6%GvF!jR2|&d6&PO! zL;Ej`qpEG)7m8Mm^jhX?eAS|1u-g1M6_AsuE6u67V)hwAT^faiqH4+`uvQ zq@k0OmQ@ULWGXppc=ff)bWL#}y)wa$tT3xpz%>XYsIoR=jZZJCdPZ6?WI3!6CV<%n zmauvN>s*=aM8%AuUBzLaoB@TXTKJS^I}F@0R|l0J8bEa+fzaInxQ_37w`<) z2f@C0X=|{y;_*ZNN0mqj!@TH2kquTb6ao ztF}1iTcO!&;My5_%I)H@#}mEmKo}_jHKStNTpEe4URvPi)kzdkVB1+p>EIz#>a*a< z=iI~F?>G2t1wb-WK70JFfQ2MgEtxN1AS*xAL>QJ$+-@)WnW+c-o2hH#dq;K-wFN)j ze~kYQx;${XqC&Ie3zuSccUJRAvV2?ZG$z(EB>&@RIYn^sDwDI`AatE7;thTubN~`pMKPGjmeyD^{L!2w zsu)6q`SVE&7)`qljh^Ug9jY!Nbjd8!x$KMT1>z}}mpNZ~V!81^eXqn75BqeSqxSc* z{=g{ZBza!Ai&H!jWaEszs7#Zxmq@z}{;5$1cj|0PXOaGfbM;-6~kv_vm^y8T#;!ot#mCJ9KZs*WK?Cjbv zZ@adC^i2iGj|-ADa379{!Sxeo%T2DUyqA9--$gV3bGp9eN}9)TtZyjSumXZqZ@y2B zRH;`4e)Xsfc#p^D3l+Hxe)03&O4%%P0VbkOp@2~of8xbq|Izn3#;RThA$EBk%+%pP zDoHF}e|y0XfM4x1)X9l6{)V&T2wKQ>&+>jwxss+W*7Ku%`1Akb7lo*U`6eToD0=ZQ zys13P3-0EC{-4g&+0IS(T7b`~WMaSQIBjJr?+ImSk;8>M!J7VM4WAAzA8<&M?iYPf(qcod|=e)&_K6>=Xgm3=tNFM zOS}*cD5$*dAVu-7RC`s`5n$j%k&ObqA&%AR*KIY8<(mnuE%gY&SFpvVX*Aoyw(V?5 z?@vTyweP|P$}EE$Oa+Eq0c_~NFNi_YnsgYn;X)#whw~bK&X4i){{h+)vw3A zPc2ldCt2*PFwO*C*Z}l=)(+zcy9>jT68#r8aAtgMcqk1p{$+lFu%zt(@dP**Ix_wh zZ`L%Ox3Z{#jW^M)2Yc;<(6j_8A)fW5Rb+7ZdQz9#7&%0{z?3dm4@S4S*uM?u$oW#| zN`e)z->8WLn(0RdIT_Tcba2kk}7@8UI@D}q9c*0pY z+(Tk;sV>||yF!{M6r~Sz5B`x+btU`GtB{?62ytU zKibNd)a)=gEn*ft?tG`gsEV&<$!%iFWyQ|Y z&>Z@0|Jyd8K>cYQ4om(<3pIe1n(67j)91nm!8+m{KX&ChKu@$1tAoc1MX}#mC6kXy zM?W&#XN)SlKk29i8@j8J-xB5r7a+P?f)8H( z{v!paa2^fY{6Ty%mi;3Ej|Ixb%jV8&?91y0l~R?YjYAL6#cDaMfC)?{*Aq?hv+E70 z-PAjshZQRjPm4^Laa&3+t1Nu**SqV$^&aw)LinJ4y7>sswe0Aq0{bi2JT&+|QbC5|QnXHo>e0&g(^B@Bevt%DH=ij2)CNt72l zu$8}hyBhz{)aEsLyPw+5uexXC4n4X4;}Cqp^C}^9p~xkK+iSijtAS4$AN~{uCM*l$ zte-q1q{*9q+?!N;f%{MdETQQ{RVqCP>pF_75Pe2=q1*z9J4VWxF^(`3A|5D?8}73( zy;>1>t_Pl*)@LHUY{e{9RpZV>dB2wT`}z*NZ>Nq= z#}%8(;)G4ht{Y^0aAM@>t*G*{QG0q<~ABS&mDb)GpxP`iHi zekzj@uj?g;NfeA*c}&+{)Qe`ndj3GgRdyba@uOVtofd76KhZf7amd}Z{m|aw2YoXS zZ0~$521F9*0;ritdacI*5^@Uj&8dI}LRz7>R9#Vt*DB-8r(7E1qcb5YM{P63_ zo@sKkV$UhKoz?eguAxaG|AAPCD%@IK1HMVo;j#T~n?W2s(UkUGsMRsu-Ijf3?JkwI zoST6v2;uz}7j!O$o5((RC|%txCi7T?IqRiA>qd7smok%hHXKv4|Gjph*KW;_wp1=` zL9@;3z~c|kfgLf@p*I_U55CjPqetxNq@w(+Tce7{1Y&nK@}}%zYzP4*B%tc&k7c*S z*jxJqm6hJmStTjfq!v1|A3JS4=6Iuz(^(C5LUk~2&WBq!XugN0Fts1nI+~jlL}?CG zWB)XK#UEN>J@C`er*{VaNM+DtjiPzTzm1@=+%-4;s_Dlo=IrSVS#pBvZCPIIfMG~% z*Ke1!uMg5byNM@uaqjA8!g(_nF`~ikbqxC3mgPlWI`8Rd-i~JSy$Y+wItU zDKQw&w6`kI0i%bPVdadHV%>q2DfxP%KaEyX0aN<1BMd#SNY{cGcBuqh{3#ch(^NcO zoYZg#lz?IESIH7}<+&ie+Zlo-_cK@akjYYQ9lyt)J(oRM*yc3u8PW3%9;ioiwEN~&WrUg7rH-1Eb$_1A$v zXT*wX47qyA#0_<^zNG#wBgZXS3FpL&zFRJkw~0J_ z&WTNBDzTdue9qURp{Tg5P#N%~j29AU|aiac ?k5qV~^|m`=NXea9b{| zFK5IFI5A`I(S^X@@*HKePjSrbc%gUW3IoiZK-dZ*5 zE7IwjrM8g=vWFXzJZdHUV!GYcLQh{`C6z;!>+v8hBi-2dMGq34 zk!a07WbX>f_3iiwY%?8E>&bTbai&(qjA5&i?&Ws(4{SZg2PnxN9}VmA1%t*vM0Qr_ z95KE9MF=S!nl&Q;{cKYDYB^X}TK1Gyf? zHA!qQ(3CijQMNAb3sq>#BonmVPBBQ{0YaR`(HH(8o(|$a=BITnNgWq2p;e{x){FCp zR)s&Ovm$iz3WD#pU0wbuNSMeKK z(FfAub@}}mPG^58Qm(n6L+OOh<@t@|1H|V0(gPQHv2nYpdBphya`S!|Po{^3)9Szg zsO^=+X72J4i>iZvat*%}$vOwd9QELs12~+Vqi|K0Hl{lNV4KBxMg4uh&-X)P zrA_9r^m~{4nuYjeBX(LubKYT^f!bRYLna!oA{Q$|3YmB>!x_BK1L48olsedOwS?oWLRq^t> ziBlXV4rQzus%G9hn|W^|FdCZ&pFLmw`)Y3LY-ZjYoJ<OlxT-WoZIVB{1u zg&Q0##m9FWs~GyYNhXk;os6>z^(lyxQku~Z>aIv82zKIwWvREOyQsNEY9<9sarOA4 z2)?0)&>Oy1G)A2kG|*$0h4QuXTQ@{?vXH-JHO1KZYDi#nFH7J4N;#w9OnCnEuEtK( zqQMwa(-BgE@D2D%Kozdj;JzK)o7^HR8>`;XkrCxKvH^rg{Ghy#4T5?s|55KkS3(Lo zM~?1;&45l`a=I1o9)k8C2tVOoW}SEuUCC3Nfyowl@fbq!Q5w#272?W=HOzT zlKvFuR9wYI`n?Oew&2_S_vAYc0tyA17Lm&EmQ0GBCiRgWfp}8`?jOLIjfdhP^uXU+ zC0#Z}eTRnKm*4J#R)pW?F4|hvegmtTrampl;NEz3urM|^s$-%6&iigB#Z=bh%|8D~ z?{fx!b{MPbn-QwO0;HB1VB}Wf9PdmD|!6e?TD_7529VEz&n?ct%{jciB*E-?n+5 zwLx@J+}ef*(oW@D5?YN)a9@L{$Jta~Ldq&rUvd7iXj5ydN9e+sFIu=t-%|@cQE4W$ zo6wV0ofP}p_O`aWQIo>Mh+XGI8}wF%(xGc;;zvBW9F3IG`Qc<>P}e>&8bnu?ux#d5 z>LXU!uExpgcIew6EfUUt#dbCfn1_2Prp~n9)Le%ZD`J`34r~ z@()%7M@XQVi0Y9`cuIXe1)jXRWxaQbQNHOF`4(j>+4!DUm&U;%P1)qUK#^d7VB*M`VJM+|&OGU#~&ofIT0Ag(ZVtnw~c{IF`22 zoz9(qUS2-+hu6*W{znu{GiA!h&SE5_>G7pGhsVF`G{Bqltw;gd^dhIC!`RuzvSr(J z1g~({)z2Jk3d@YNNiV@2wJ@_P@BNvoI-b9gTT*D9tvCfa+Z3-;xt&#fPJ8tg9+$-4y|KUQ^r^I_ ziU%FBZJn?0{QlM!0yYt_jWmMAU}qbm_&j=MG@DzBKT<_dZ~9Ai`T0YmAs(#g3dC1o z^L$=h9CnEp3XYf1-`!OooaL0xcg-IVztkIMBD;LFbP5Xd@~ zcyIFPfO~a6LNsP2c#@xy{b=;L0D>0qbIq$7>9aTG6RWY!yx4KwV+pePAK4uc!Z&OXx(=y;*2om2nCOsV8RB9)vCZRIL%sxIgBpuyhg%-qc zqe<9rzMmOdL~xPG3dV+@f;O9#7wi1!A3k#e+JPJMrBS_q4BBqjg>+}tL_+I4yw(48 ziIe)x3`@w^Y0>Vmm%10`;46x!Ti4^SCvy+$o>>aee=lyZF0@s9?265CzuogbhXG*j z$s&NQOzBw@#}%R%SDo0HgA=&W$nUFCF?g4UIwQVY#dzC4fI(GqS|WdFH9f*-&&}kW z+J~A?T~t$FL3uqqcLbC4;cV*|zhH;HYQ~^resL1n=?Ftg?Lq36_u;#I=Jr;k1&48AD(R?=j|wQ?7n7#MU2AELW~YV9y`Rnjep8%)*Ap{&Wr-9A?%+omz3#xC9hu1xpd zxzf2R4wP(*Ae4)zV(!q5srVHIynxQNihHRgcmLq)R^-L5OBat|IIih?e*2?H3z+*3|PG(}HNY2*vSPh-(LWA~0qVNrRY{VWSboRptU3#|z@4FFNL(sdgy?<`OeCu$#zitm5)1voEYpX`0w$Ac&$(y&K!1V%Vm+0$xJ}%#z zrzrx2h6O9WqFx7>KF_q3RoY~WIkRZCg_?JS(5IH7;P66k%)I>D3r44JLaxQb%Ge9A z3zd7^{~RjPp{ixRk8#Tj8oGYnnq8iiyxcrfgL>w>dfx86Q=qFpJ{fmQHdp!DMb2WC zS0u%9R)(}*)d2nF&mDHkz3^dp zbPL(pZh*V(wd7pXOd}}QRunPwz|OES;fD=~ki_|8!u{RFdyb4M``p$g*`Ciax4+ex z_u~}@QR-}KT$5<~A2#vsFP^FG6NLH?_+CT~H4E^C!_|(2fMM56)iVixwwY9SAX#E$ zL+|ov&G$``@?CLqrS6_e3#Pwg1*o-^I%}P0$umu^7bu2Fk{(KApj(##AOXD>Bn!6R zsFW~IoUA(gkNdL{@Ym66nwL|%aBUDxA}9GJ);QQcgKyR|w2N=4%1Lt2soK>znI@Td zHoV*3lm(IjqPS9F;7{Q~`LI`3EWMJqiT1VWv23=gc`cJaik3>?G=T6~NcnM9Ko!rn zdnb;Oql%0<|KvklvWXhZ^HJ(~6MP>*vwD_L0)Ph01 z;TK>ubHgRgcbauVW1(+yg=ZA+LyRrX_;FQ26{Y&TMaLX!eASQz8II{x<__;QKmZ^f zh3AFcA-UTb22=b9O5)A%XhteB$4<0HcrjL|q_=o{Lu6R`WBzc{oWc9syZv;0jXs^mNSw^I6)!^A~#Y~4Pm-B?IPTOb`qstI2 zU_!4d{q4_%F9O<&hgi3cX5IQ=Ul96@o1ViVYj6A4yIPz`eV~Bd#!o^Z)Y$ucC(eI?ezk6)t5ZwLK0~qPIzsArZx+ZOOHdq; z-2(YW{2WvtUERCYks~(eyNO>fGRKfi`HL{z!{x$&fyYr~uHGh-gtR~AOQHTyKm-7Y z+g=Cuu>$#-&sM4X zt&Je0n`w^WXUgVTCEqTy8+e-8B9e`IX3H#LOFXzmO7YE9h3j69T8Q(g<=CyJJ;V35 zke?k0vqdm>LD}#JsyVUj3SZ$yal|LL-;2R*26mc!9dkI+#eD$V+ncLiW)Z*tFt*HL z;3?1T%U~+)C=ni$7Ft~P9!}73x!#|7t?<0Xy}|NPlP)W6BS>8|Ki%%h3T@`e8BS-! z&DH18EB1eR0oM5izye#og09?U?oGG{H^tsvwdcPlFu7vV)*|>*xoGgI%?=+_mGOH} ztzu_V9poU^Vg(bPK=Y^Nio`Uj1~vZx3#)UT_^#)>I#H=YBgr7(~6`wzk%O zdH07Xk!t#^CFOy%gHcuOZV*E_vKFenkU4tvoq@32s(15>_n^C@rF-zhqeV0uTyS%v z)lKQ=b5xAKOkzg{pNTQlg&2%ZDMQx3yTg7+Wma#>>wM0jnZ&|xiFjVD2bW}}$(rdC zeNV&5?CP)SDCuT>p=o8Pn%VhE)Z3SMhEs+?pSdV?EifWUDer*v*!mwdLm!I}W8AS& z^k#A${s49op=SGz;9wsnPuT~%KiJf3|CWW;#qZC$Idv{6qsNbM1p>HUvIW=ig#Q;@ z->R|p@i{Lx^w4ZWcuYMw`<%~G4&IVm)l4)*UZ3f_Eq^B-q6wm>UXR1dkcxC{3Or1 z=3=Zt-KMF9XO6e1R>I1WC#zC5xKW3_Fwri(-%q($Ks~iPl!DQ9>-U&W@}ncWNBkXw zXL1R|BXeDd&DnoJbW{(~=+l3T(Y+IB+ZbKqC0Iaf9dtmp&S6Ay+_~^0OGEWsFypUN zPx>m3GzU@h=1<6{%*mk}HRUm{PTxev(;}t|vfTGKEX?UtA5u97s0ksd)qLdnpT^hk z^vSNF?x(Bx#VleHh4M`;EQcssHBivebmR_7uI5{?d@QN(50O|Q(+~9(-|$_k_)^47 zbH0sP#fRQqz8T)m0GVs`vw-;v=S_OTxMu2xpledGui5)s`S)B~#)*P{qGQM+WQKk_ z*!anRe0%u?fKr6&p~>s*opKvJ#Z2|y!1(<>)dBQA(vzW9BazecAFICxagWPPNFBk5 zh`4IlJl+nT1VCy>M$T1|V=4b>R<$<^Hn2HTzV&rZ(7(OvNeO+te1|mny~w!?sS(P( z`78i67;Gx4!#=Yl%8errd_~R=lN$+gf!1t;-^^CSt}DVIW8NE8sIb)++?ArlS+O2B zKm9eF1@WJl`%?3u2C|qBz1wSm54)$LhF=48N-^M{m)_Om8LRKa!gUiHzkFX4p_ky_ z-c|b0>)$%fFW;=EOKKOQfZ*Sa75o<>>MWdf@VA!Ml~hkP^z%zds0bw)I-fK2{25n6 z%>Em1xBr8;dx-ZBV$~gH1^rHfr8kR$NaT*3{&+lU2+2Vm5X8n=t#m2*m49QQZuMSW zi{K1p9q~>`vAKo+oq)^eN}9x0Ty%vKOtppyN=zJ5KxgNMJ)|-hVPUg+^FGV1y5r*^ zU?(e9=0C4u`f!<3vlHcEyj1n8Afmg_e?spY_FK>!{%uw{QZIBT55*pW#ogL2D|{vE zm|r8=2FGTnEuL$do#x6T&0tSP^P`51&F9W>*s}r4}|S)ou`a)}R-1v*X8zz6AnP_iVasR;n_KT=8-O{X3d3=_o1e&Aoo7_sF?b6;ROr+SZdFMSw6Us5~>O7Y~rU z2iKY6g&$4u#HaH;;#;xn4m8+sZ;xJ>ANmDC@y`@pj$2X?Sd7)LN@tz$&(*%RY-yb#NMIEQ_;H4y=&%`*`cM;SgweH z6+nS}WF+Y)ekLL+th@Kxyuw_h5z49I`=?!9?+7>uMl$Mwa*8;40~7(n97%3ZNyAco zv=Q=~06JCc*si0AqZ={GFMN&8yQ?Xf#*HY*r>3zew*h+Zh&lg|)XbD`WXqhADiN6+ zAR*FP9)5J?Y)+K^u@(Ih%YCBSqG2DjwB0XVLWc37ZOVb%cA*pV@-qFKoRzhbaNO~F zm(_tbVz%Ii8b)eh{0BQ*dTTOG?YDk*#tJUbYqPpBa9Y#Anu%Tq<&6U@Da_F1mB0W- z{O0D}0U~pAnucp)yrsL*nAD{u?6;aJnAn7@`OKL$a`N(^#^$gLvF5bE3QU;US!1?v zfHf{`rNP>HEDbq!j8=Q<&-ROC8^wBj^(I&^hpr`8Z${sWVv*qpzIUgjOhU@b^r1q0 z6E|DGW%E7JBDzy}qX{RQq)470H>#3SAQt6@mczsuQr8&0L=4&{=*+6@paxs|z5lnn zJx-1Th+}7%D&MRuJcTQgN&vysPieUVX9%4B%c1C`{tUZ9!XFebGo zqrKFTZdzj=P2DGionp>)%+Gs(>>9UEY(k!?t^JO`WK)imdgyEi3(F zKZpGbH12833mK)n4u5aLIbCw?#s|4lL*9+*)cMlWpl|hj=d5GyucfBCq1YSlf`QP# z`K{>6f@EUJ^VeH2UyJUwh4QVT%d%wbVsyWiX#cUH%vWt6PufgT8&Zo3BT*@M5Z8!$ z2`v~COzD#z(c+Q)v^fV-6-nh97c4$o`i2h+;rsqV_*P#9F{ZIS`Xwq~_Z#7MuAPG) zq1l!r^(qO6REo8~@wXx3Z}&B+p$5t|0nDEXDg?7Nu!i?Fvy3+{N!;+QiqLJT_X7rU z32SY zW+o$(%ooQ#Og^Au_`;ntzgbU;p_!TCgBuY8mdGliYi%b;-C_OHFWM*pzm3;^3b^(;|fC zVmOrjoIasWOx$#GaB=OqJoGIQ5)H%w_VLQ^V?${K6t6#$LstPaqev6IgHKXe&Zx&< zVQ3$?I-vb>#XKYXnPRIZ)2DF-gpWY$%B6oZh)TSR{c@%o&=Uue)9!G=T5F3pPcyI9 z;|&>OL?WEggr;L$`ym2>JuWuDBBT)59cU99$eMtw`e6G#rZ=s;v}BKd?{dvXAAO(^ zgz2ZD8a*)!`k|8q>_2Ij!f=`?6UMs<>3) z2Sj~2vf|ac4cgt0p$hBG9B$(K{xc^JHsK84;aS!zwKLv4Pb>9`QV7scLOY6{2(=&^bBf zj9I%$j_8XLHNTa;kECKqxq8|zjec=dCHu(LL2;%*02GLe_tJL$t=oIPfJ#o~^7;wo z@P`bw;(L~|iEjp;z74HinTco0haw_9*a^vLA<4mDFg{-q-rVgVe5@Nz4MSySA~u}A zS`J&5m;J~N%hfN;tQYWC&^aiqkaWWiG>)E>R`5cOQhT~7)vT1ZAdI!%v@ko7SNX_d zU_zRX|^^TQU{ z+#YqHYQHwcWcFc?@ipl>e3XLw(nGLG9RgeD8J?@_hQ>=d*xdQjIt&MpiOq1jzFcDt zwKikxPxd(aLy;21=kC4hVB_N%<6!iPMk$pjN!NP6(0Qh`xu=sYuv6?)5(A*6 zEYZE?No0p@E?^J&;s$50(guDB7)+{aEHTTme6`%#4#8-r4<4f#$!P^u;DQUvevEPr z%BLhUO!%8h7x%`EWEJ_Jtjx$!khLS!3CNV0H*O5F%u@@Ci@h7K(I@=;*HvBGA}^)q zMLs|(T@$d77%6Ib$`9wTzyWUtaoiYTVt{TR@Ike@e0V;bc83!-SUPG_P&QVRMDbbJ zKY@!WU#~YCbD%jAc9&yOJ@;IZf1q~`tce6mpqYvdc;;%fo%iZG9+vOFIbEc8HTnCS z!k96KV@ux>a4S6PX#bF{YVR&U`VcbgGbkJ6VC1s>0oFW=s0{1X0mXDjW81r6(NoTq zm(L-;>VRatdDPV-QCvh;k<595(TiJdvhZZDzWWuftpRvZxDJ# zG6d6|b{$#6Jlsv(hqpVK)VU;|N6zO19KxAm1A-p^+ez=rQ_-}zGEE#4UW?So=O2f= zzwgysh<#KT`)Glwzg{$4(QvmN&7_N9Hz(8RMmuIVUZ6WSVXbz%1DJ(AyBs!?*EMRA z-9QNeuu|GA*=V?QC-{7f(r|L+He*dV@Gc@gF6rvkEy9<6a$}nWxR5P?rP^JTPYDZ$ zD|CO|BBUR1XPQsS4Ox`1g*{)*o&A1(Y+(t5t$~ZsdcY-rXiFLmkabB?iIPyrn7O9| zQYAdx9xbiUm#XS_{p(a>Pk(61E7K;viTzN$@P69p{Q)QSZ+5;2M1wx08EE|K_UxBq z4Ee8fq$K%=p#Kz8zdbN9EN9i6-Rti}A#;R@4p4Sl4lG{fgdx@i5b{`it!IAiAC1*DW;j12XCYX=(a$+9i$BWYm<_co&NBc7Wm=H_Z~fI@M0d`=2+Uyt`TCdiICXxq$X{=Wvf9N~e%(Z1; zPTiQ}R@FsN=_u)RaS6q^yMK&+F0$w+OVM9S`0@#IuT&oR>eg9%6j~3)c8Kpm5Bbq6wx^DudW+({ka!+q?uVka{*~v%tIM(dQzB_%!%f~ z7fbz=q}~YwHcsc2Hd6XvT2*zaPL^2{CO*g3^36_))*kl*o|TzVNojy;X=YYm792~ zW&{K~I_49eID8mL8#d}@nB5%Nqf;vG1`g4LWD<25TnBEH88=cQ3Iosn#&lHkWXLJ)S8< zR1qJOt{=O1kC<+s0*2-e8GGvXo73EZZ|DN5AT5j*h8Z~DtijVpnUYd^6ku_gsM2(D zOnP2>ZT2EK{f-?|ejsX3P}55nOs9T+ynIQ(R;bH#z-d_se8~DgK2%@f%s&|;D$;Yc zaa{BC4@tri3r-q%JEF_`k$|mt3Iw$QvQ1p?SpM5R5Nq`CK`wM^1o!WQTbJ1(`*Lev zW&P)WiWy-0#@-xe?$_g$j{5mKY^dn~%yW+9 zZm(YvzGT8b$JcxYf1gy${azltc?z6io4SC@V=i2KT|H(@xG4Bh9i`hcCD?J5sl&Gq zc1?42y?50&qK#sMkChuXx?I6LKXxQVk3}s6wXt4V<)X{d@ZmvjMekfZ?T&&R#BlGl zF$WJQOu~PF64Bq4zV!~vkA~o%+WUsr$qt~2m3yN5&y-bOi25BsiK<3GYLNS)JSsz$ z&z0s`ZqX83i`}$&IVW5tf;%>OlZ!RQ*CELk1Tb7JG1NRp0$cFV2KwCp=y z-b0BqzVe;T;d-QOo7zpICmL~gl`2e$3SFsUy$0(_m+bLf&iEjzuQuXfGJe7=|NU(0 zY^y&l-(Lba1QNO$&AwRHoF^&YgWl-uqGrl~cVAq0w7uf6{~_^5o&6>8U*Pvq z^c+4)!@ITJ@!|Kw!=q{>ueJ#<`!LjWcy3v8saN_Bgk7f3rE>Hx^(erWL>fQDLf_R( z$KU+vW_gDWI9=i4?rR>2Thl-^6eNm%owE1(%E*xY4^~0dLWpiCBO8jA^n*ek+ND0S z@^TRF$~0_HE%Uo?-C|4mysHL47of9$JX>e@}IT9eTe{=Rt^#Ox$%2BLoT2(-s@Xt+ka*0hHIOSS=CyFk; z<{u%3pLBQBNJ~>GXzNfj+nk|BOglWus$vdXTp0Wuh0Rji)Kt-YXqlM{m-$a?U3H-1 zg5Y5akz~ zYHkY|0?Q`oCd!SkQ>3opO4A7wY_@Fzx$eBtb$_F>z@oEv{eV;8z)x2{(_qIA_2Fwi zrv2Hb0*n?6TGh(KRL5g)dCsG@g<>F$@PsvgQOAUGNOyUR+I@x~Y$)+S8Bw(!sX^y7 z8y8*U{$DW&-}nB-AWSKHeQ5unc(cqEHTbw*eb%i9%lF%z%kAH#D4X=~QNNM2?(~Q~ zZ<{m9jJ~UovRqO1oq7F}fH}~|ZMF%|Yuq1q&sYV*cuergC_6!@B@=wQ?ni;7$O{ZgfUWbBI2+UloA zqLoFC4@1#hy)o%JbRWysV9lLm<-H?T+|y#-ts}O$!PT~NeF!hW@!kVu4+v7f7D5gt z(dwAqY(pb={P=~G%WQt#{g}nvEf7OFk&w!Y*LrM+%?B0~=DbT!-tc6$>(zuwi$zLr zRwhSd9ZS9k9GQGcIA)bFrm__}sT^SGtMhenX`K#-_4O|{0ETT+^R_X^{bucVkYr3!)X@2g@}!Gd3^)&J=31X`e=5N^I0^Ogp zdKb!OWaGVZo^Q$JsZ9~Zjk7GPf8NO6dZt>DRn3&Wy;;*-i0Y(_$7PSGrS+Pc#MQHX zem%cdwE7j5s(ZjM>`Ynv)RJ5r+WkUV$^-L_w>}?H6J4Mq@P38!pU3i2ZM=CYxH!As z3xu4FL(kRtJlySNR^9F@>5NEozOWs)=vk1%d-H}AUieU(0zWiV>dba=RF!Q#n-@&q6E`A- zm#BLuh>K0`t8KiG&C(DO#XU=9>*@eb4gP_9b~ydlR6*#@bnK@p_cp)?3bddoWIvdIEQ6yYrcno5OwSy3Sgy(EGLu~`??yi z44YZKo2ua9$y*4GnJ9b0M)d~RKvvSc;%W)oSTEfFm(l+Li-w!+$n%c;;yx`GNCITN z%_V^^C2Xz3BW-^C&$^45HIkIdnjQ-!znIy4Vn+i1q`I92u`=@9>fkhFdR=JPVk1z6 z6&tm(JpBU|qwmz@QGtYslb6xL4oRPZXs_;{bWvkY#GqHF!1%u1vcZaZ;T9bL;rM#* zFE7A0HSn8KYdF^4New?){{8BlepWCVZCRH;Is3-nDQRq=V$mefgs*M4t?;oF`NRZ}P9bP~@B;HvbE=w);T+Y>wI>;gdMw_l})1QW<0Bi(2B1Yg z)QCFf4_$p!Q>)>GmKywVB-mgJ6+hM#5GGFEvfL8qcfik26iqczQyKDp2Y|r8cAT+o ztdF4E*F&b=(hf^eR!$*e+Ad9ssQ%+6Fsfcv|L-t9s@~3b!uc1*cP7+4&6VUl5quX1 zUIKlCNBk7!$sCUiX}wtD_BTNc^d{+&3RI2b+!YXURDQ?K>9dERIXYd8TwlllgwI9L zgVDZYIk`aW7e=ZgV!i2{?@8J?(MYKRaACy1MR+;)i<(-eaWP9 z_(F-!SH6FNbK6hOyyfQ`K=ku(aQ;W<&G~9em6B^+I|YB4_$Qc$=Ls%G^T3T+siVaw zaBzZtDY<6W4l3h)?(5;1s=8U1EHzG$nDBKZ+Ji6>gnMJ?QM!Cm6Tl0fNAjt1zz!>l z>;CYnw(vkdh<@db(?T3T0XQ%o{Qi5`J+0;acO>q++`1(Xn$Z+BU~OkK6mzaQ#E`ze zi~zXJIxL+14tCeR!x^V7Z@r&8jp+WdKRM)YQvlS$4}KYA5r;1ylM^IFFpmEv!xIg= zQfKk3m^_#9@<#%rSml0B@s(R2eYi~^ol9p1S^j`TJV7n@$7(7FG_L&yTM;=JXhnrb z0(h)4Y2^fL{P4?3$pLB4(DN&%OSHcw{ol&|mh|zbNH5?Z&}?H+@@(o{)YuvEoe^~5 zFL0y9Ela(esL?K*?}CpZ?_mUOn3xI}cF-#bAZVi(QukMxVz;j(vo7e$ za;hYdtsh$S*XfHIojw0VKB+T&Y(>5)Onx3D3SQhH3wDvA3K`%0cElBS1{{-D=yeSI zVGtN)<4pk8rPuLgAj^L@L}sTov?~SKsIuxc?3h2kH9uH~AO8os0RDfa|0)kH0yhs! zoAYhB?_=-j#%jqZJ4;%3X>VvQp^B!EEof2H%Fq&!BnjT11yo~ZiN37M*q#LcdO2{|`BO+N z6e=&==<$A!lNJkmZ(ZQksl+Y6ddE&U>gzFJpOLNsGl2ev7)XZpTym$U`=0$v;a-&5>w{2wXyIStMZOkUAOLriP$aQWtB z(WnW_%Wob~KBf#~EDJx}yC|~e2>G`S{`+4Txm%ssqM>tj!1&0Nh9WVYC{^X%|6vVk>b*e1Cp>WSU(~@&5VFdqs{0R&L2R;!vjt&`<~$NfHuO;C|X+a zbehRmzQnFlox2|IMUdGePZyVbLM*%qcf!4si;;TUZg!hbbywhs6~ER)R-9Li%M0!B z!5=s9@VfymA{$~9ZkA^rQN1MfNma*F0Vp@33g!2vE5ag_H~nwxJYSM=YPl=(t{{-u z&hbc?1IC~VU)}8;u~9DVD_0~mc-61?ODT34emET({8YUcnx>mJUYEd3c$(fDf90qAP)mOw6CQ`?sL7)@8(P0dJg)idk{b)(IQzE( zE}O^ls)yyXtbL^}&N^!gvHnbKW*ZWv(EZ@!ADBPWLwVkMRM6N35RA1dqmYD@c;Nfe zLOy?bWkn=vL_X9ll9lc<$nwueL%00%{zE@<{iSy$fYMz@g!kUr043medqq4eK{ovx z$O5&P1=_wD$oKAjJ>c-eeZ=NnPoR+WYOa5ir}dECIrayZSA_4{2|NGgPr- z&n|7iAG1R8uS$}foBiy(DrxGKyF+Wl64HW9&NAoI{t`+cu}b}lmN`O^^cO#@VS8Fde&`da6Zw^qfo9{5oZ zyR`vfSE!>rxBWs|iv@gS%$=g=4Ff*)ZkJs@n9BB6l~TD6bc@?-X5Fe~86v<2>6a2& zx?$^|$Miv6VNLfSZm>RwsAdqqZEqX{Vd-l$V%s$DXO-OI4oq0 zB6aEAnm)Pddd!ge7NF4=oh&qJt`5+s{Gw0_^%ojtXZstC?l%1!jZ&zebh(c5*?edI zvCwwz$dkVzDKWc|LIPIvB{A$O{IGT9Vz7GeV%ka}mR!;KUj%425!MD^%QFA{9DAHG~x&n8F=j z{cQ^8waNI1{Bo<}js3%~1V5-+q>qx$Zyq>B#jMApKGar#&@jNZGKBh6F-Xd%jG-r`XY$h zzB4im|BGsj#=VfH#{N+edQNOGph~66``!ly*l6Nl5Vz0i75|5~c$}AvP+O{TwEmWA ze2%!xb8M@7jk=v2Yw_8b^Q3-prWN;`SViC_W1t=nz>;e~Qrbt*qn5*BqHBH>IZ-V9 zg>mook*|5=ac$O67}y>PFa7h9eOMxCzy&bR_l`Lv)3OQ8YM}PC?a5W4f2DS_O~|4d zIk^>^7RKk>A3%c-?%$P9%mjJhilfEz+Vzkdx*9r92aF%y8p+f8?DLm6d_`i6%&Drg zT)14AYT}vel|DCdaWv07`%>=~M!oH=di?oJ(J4^s zPQ9#{-siKjA2e*k7Qo-At=#O_AUE5)|2z3h9C|)x7n@!(u~m6KBzlF{u?OU7=MhP3 zEAQCO?KxDH#Ms5w*7zXqzB0CS#XKH3m#TFV`LSgniF=?bIqfK=86|*=BAa_7W%n!k zakZbieBLi3lEJ&)C$xRd2jH5q_3w%ZyFuY$prOtuOrtoNLaqiAq4;Vc|K8xWDmSUC>{6&eSpGNy@wb z6`m8eJzW*udL2DK$uQNOZAZTKJk>ud}AEnZ0EY9nBiG| z!?T+jH(^SMB4B zw%F&A?|IJ;hIu@2+rny{>wjalXFLCe)uK^IS5#%GapD*4Qhdx^?D2H()u^bB%gACL zJHFxw>1IJpl2GwnU3tl%-=AGn5-B7N6}{!_4rH56_TGcCNv;d}*V5n5+fLk1kU5DE zNknlVpLN6UVq0eDVu%N;xknk=!9y6e_mj@}djz`Rl=9*rN(5Os$4aRVe20hHTj=mM zt%QhTn_c2(KC?#8q4mC_E9R5`X(=18XH97cP^QDBAG<5hn4h)0;%gFn;6pp@9!r_T zk@^I$Tq+Mhzft@reR*m9+0LA7dbgupbUbl#u^u2Y(*dd!X=a}BPRDf1cI|z~QkBB{ zWOu#|vo0taa!mPQC>zU_ePpizu5RsI@d!)aQ%pm8Y3fv4Qax-#Guh-tm;v3FQ@Wau zHhR|RgM?|tcKFY7D4((U_2}|_MCHmXL`CM4cZ&TsEJH0~@#H^9*_Q$J28?5K}3In5O-=4cJ zIYkod`L(%&D7b4A4mdt0&V zAnq34Yx?GO^I{0z;Oq-@?chZD$kwMmF8vLGV)_9DVi3A)~#%8U2ex-P>OI=a?5aAVJNZBzc|1|;#V%J|7H)s3bVPt zweJA(rm4mjw*zIJ*&%#CXE_K!ZzosE-bhb$_z@=~g(;nPUAs z6{YQS*8|-72^jMRDX}Ua$D8z~(?vh_a>GKSp)HUZN^HEMd``Ui8V>0GtXe|7v2A51 z0V8gGu81!&Fv+_1`gHqQnpXMZF1}R1kT347Jxo63IS(M!ryg`EfGN>cp?tdV9&eKu;a;9C)q67mjf()Whsw zb8HlZn+44()7~JZKpTG{kNzD*{;k_Ja}yyT*_i zIlIfH3P$<2ZmPBgg!|^IXG;SHp1#JHuoV*L5uTSQp#0xkIsAN&auX6C{ad8;AwI)H z+l^qAuGuL-F?Ved(;uvvJvcOpu#9*O3L%T8+0jA>nj@^TEeP#pmO~D3%V63E#k1lC z+||iVuN5Lw&yKvrP#JT;M-@55z>)kQ(=@^Y013>a#}vjh<@_s;F+SZ+fa;%o*$>KZ zdJm!Gpe7NHttisxvrBb!+1V}@hr&8Fi0gtv|K7UUdb23TR=#I)aM_V$HE|JE~CxaU{e5O1tX`DP>r&RAxe<3a9ed6M0P^$^IY zQ_XAbA9Gq!0);f_eg)fiyBApX?som!nl$_V*_z_wOP>Vv7_4H%M>^9AbBg<7l|vl? ztH=`NF=$1i{dOgc%KQS)-Aavo`5Z?>{uR)Dx17}^4AF60h-wGx9Nm-Mbe!DcCM}E z{|RJkg9aDdW<|k4(7S&Q`TwsSZh$&zde?>NKjxUT=3?eO(08+)t&vHM8PVcrl3h~F z1&%`>yS0alzjC!5dt$b#Uypw#0)2cepEE@!eJ|uQC;!=!8jpupUve@^?n+*@EK!;+ zJm@qKA=gZnkyrR|;%#i8;UT);l~H))9qY-&3js@2bKZbWnBTGRSLqDZD}Qyz4xR(Q zdU!`I+vmTwvHzWqj%wZ@UlQrXH$^U2cwQgG$~cwSh<7}tsi@60Y*}}9tOg%|Jq`Q3 z*P&cZz&w#>e7mwpEl$gCx6DC_JVJzw6SNw+ns`H0J+Ui``>)@7AI1JAaNFX6;_>yY zx7)1P4p*_4=qsl5^#v~@amK~(Ei~T}0{wapkEAGJx@EK60T=Ua@KlBi!DQaN>&8qt zcGsR#tF|kI84-iyhkFah%4$jimaS=}{gsnKw<435ZD{0VqxHOo(_0=;P@7nIT>7qo zIn3AB9Ir>1I?`!Fwy_!~Thp%j&$z;#BZ__R=xFVH_t(wngLWP4Jf`rZY^2@Uwlyio z=b**iA>!T~t8oXV?qTC6lFlKx=!hiM_IGQ=eT?f@51jmUA@jqZA+_O!nJ;jK?pNv} z^%{;{{`f8ut#-5OHlJf+ZwDht(0Xo~esvS~l0`v-F9Eg%K36v(G2@czTf!8Oy-$93 z$#Q)heH6Q{DL>Tf5#DM$m|!dnd!nxK>+z1Jf)13C*JE$}fDBeJJZI-Wmwy!NQ6m|r zC@p!pxYwN1gYKp}=l#dOu5fPry33x8MLJaMsg|gcm&^3WinBp3-Tsrgo$}5Q*M|Hc z-zzb^9Z7;(<%_-k|87mm-v5WW_l|06d)r0bHryb%Q4nde6+r|6=}kaIKtV)MdWrNB zs&q(#Wm`asNC~}n0+CJv1eK=rn$RO41PCnzNC@pLiu-}~=Q~mG`fy;nINl+jzi&IY1m}JSH}?MXLnxB7)&gf3&vfF~>j*ywY@KW; zr^}#(V>Jl%*KiajLBi@+SK32?Z^f`t%W>T+-lVDBobeA8LqhM63x5v~)MMSTSrIA} zYZS0GV5*-wd7Fe7ZV+0Cm$3b3BT*li)`ovu>z$-{KULgbHVt|xRFupLsxIqxH~5|f zm4IY>uKs+%?NYf|eZ2cjWU!Gkk8csL#OLH1puS|0d~)Rzh?K6p+zp#MvsBa)xDsBb%9Xx*r|Qk#cvdX+ZI8wVGc+@zyi-^^{`@tMz??+c-i@k}$U|xt zZ`qt+x=iDkX*I_zarTF&PaXR;cR;II>=w{xhzO2_(*oGp8fI)N=C4t#)<4@RMlOZ( zTyrl4nP50kGjl}T;({DBoE5()`2J8zjvRIq*x~l_*GES#0*xPZ@j2bE=NP|DK;NYibawgP9XJTG`-O7UQP7Ha>L4B)N^zCa_W>GeZJKfn=`L$^=3x6LwXblImS#!!xWj0bV#5ig-F;X~X2)xy>aet(QEw z4cNawG*^0rDAhI@mC!f`YR&gTgz-=+Z6VIVl#}~v;1(Ge(9g&w!^i2ooL7B7*&Unq zH@j|<%JmC5Kne9o@BIQh@$7pIOQ{*A3;|06rb5c1^_r6}VWv0xEr0&wH+n<3@a7EK zOr1p$>tM#hn7Rh^Ui3yuLQmzQ)2SdP#O#%IHPF#lL*jO0&fZMdnCanFt8#@rwScD1 z80BptofFc1ApNH<%M$*=%y*?g3TM*2mh0`;y1BoNW`4sVM~4}Th@8nyU*)Ty+6qZn zU$1L(Ji3=xr$aQK^|sU`_7W1d*JYi{z)={sU%3GIM`fj;>SN)aRB?TNfh;d47tZpE zg_-X4b{#%Ylcp2yoqLfh;>zr5Jtv~KNgdmX0$1bLsnS1duLgu*An9flDpUt>PL&NXC5$rlrC%(vx zL8ki@^{15_gOSPsiF(Z&STf9|Z9{((>`@W4||K5|kp`fTC3DK7e z48P7sUeZyV1dy@kwrUVn)(P6s@Z?1&`8TVBG^02wdT{#@kAi81>r#&`C9xHrY8#A1 zTHi#OxY_vGO)^Ofpt!utmHFhe0-Mz@5u-SAMT;}fa5QNrDTH<9F*$oSO>9dd^ zw_wWfb7ypej(ZpSNI`}NRa0pxf9mv<%SoHYir|-)g(;kC-x_hD;4|e!Au)X%zYQ0Q zs|mjDwSQAfak%4T0dwzoyY36DSU?JIS-?v@)BSJdeb>!r z2QcEru^}jSKJ?NQ^o0WB+j-Ha!A6NV_xhqTu2e1o@H*#diW;3Qy}`DbwrkJ4(|)-USzg?zh~mwd3(w zozBG6j&^d_-+qASQSJ+?qqe%3O`7HyhohL0I?_8M{cPar*kUA4+NDiD<@3%asU0?S z`KSya!G?3qtpT?#;_P{Ps!8J-w<6R&aN2291X3rnE|C*o+%l}LN0KVcNL?G7?apal zYA=2uR0^C9oE+*=+|pT@O!0scOmKuZ%)Kw5rD%gN!k9sed`dy2UM)6yaw%f!>$YOP zRU&!c$%_Z9O|%$xo?Wr~#RR(|U|hK+TAB?iHN|@6(XKFNVpYx(-G(wap6Up5klz$v zkNs%iVbSch{@AHwD;?m}DmT=jw3TF3-PdASO;b99Lfq31KItYe`&uSdtNtIN)vV#s znXg6yz~pIiPsHlex*t_@oCI_qq%G7hv;?lmmaJ|1)|O>xh7n$ObIq<1hgOoA8xKLs zumtLF#YMjVs-nTS5{P|9TKf}G<)``un?zA9{%=0(QSDFbYl0%O-uB)Y%(RX7mh_F+=doTKig>V2$+757Z-lD z6R_qtE7Sr93*8V`k_HHzxgsC1avf{A8?l_L9L}C-Va7kR3J0GXj9R{}*A7OmL0$Ul znL*GDJ|&OyZJmiC3)78**!Res@wd~mbgPZRTkPh*G{#SWqn8Nij>_99oN;z%+N>`0 z!R;hy_B#PX(4nf%ILj@?64SCeb*XTVw2WO~oB0|0r%!Jk+ubo4ua{5Wb4eIgAgZe3 z+J)ucG}>F-ii!>SG4V1O0ZYWS?(y}k%%BNy(5*)TGWur0W0z7 z5nxd5e}v{(wc;}p87P$8CU*b;yaoOflYV*qbmm9Uy7=_Q=BEdJ?N_7JC%@^HOOmK^{Db3)Ps?qPXDKj#qr7cm;*Q%py zuBREL*128)Lu8i0jw8IzcE%PORoNVCLP?kNcubn}QfyAd3uCYpl+GD-h!EY+-+N%g}=>6`N}fxL;_gnTI#NFf=2 zl%&^S;M#W{RKQ7i(||h(aj{M|YUQ!s6=ucD=6CNs=Tl&KKG)yR`REguqu$EYped~I z6v{5htkaWr>v1ElRJ6RP-C?4ZiSAh`*qZ?}gM-G|jZT^l!$)bNWT$!Al!Yxgw|z9# z7*9}BoHO-KeUT!kY%};Gr?7VZR{0VEm?Dc5&aecRMs#84)Ftj1)1^{o;PJanqAQdc z#DC_9;8!mkaPl!l2rl~Q8nhD2==725Pg{>b6`6#emHMl2#Zm*^fsKqZ%q5<@x)$ys z4DwErk-fGy_KdI~3pXH&;WjzSjUmoq0I777{Kh23e7X}2{%QBHom3t?Sl-b!CEuZT zH$V$gU+9SMTuXFSQIZLXWhydq54_#1W(smOmf*jtm>ATF&p?0{mUQCwyQT?R4XVnig$a~qE z&bbly7z|PL^F#i$vt(|LK4#(*JlW>pTejj})CE1vg$-4A@mvq{q*0*E@gqx8p*(x2V}jvmxoI0Z=J#Y>oUOO#=EhITl9 z7)>TMQ%J?t&Wo|7-e?w&kZ7jM8lx5Fb)1WE5v*c|6l^d$%Fn-3X>q3gJ~V*% z&W28G+xKjb`tGr90qPx-PFY;Rv{liDl;am_zQ2+TJy%T$?(K*@&&|m-&!)ZJpPp@P z^<9)8%+`+tjPi|1tw`ixsrd~an-D9XuXYXbWj(hk?j3&-R=*oG%Hn2o8gTM}ee4X4 z%+d*XW3^?pQ`a2{$A*ktdgs^f4bQB#zTAD4R((UR*mcLBLWcjXG%bx;rT(F3=ze^lXS-O52qqd7o%bB z&Z(o?vu|7BHv*E{MyyJowpD0$AT(y1w|7c5Lq}~?GN~UcG)^(@v6^vM8}}=Xk=2PeCcyeh+P5xEos7-% z*qK`cYtMWq!%()l9{Nu8hr{5S$h{wFXa!M&1s90TlW%D=m6+9a`)W!ArcqaXF^ZNR z`Ojb!qq%BJ_;5DuQy35?av99+C_0-kXJs6QqD79*4RihE6A$G>3ul)*YF}2LrwDj2 zBrDOet!}?+ozH#s4x{Lo*;)H4c!O)$YtH zq#WYHk}HOBIf?Z=%OJ#lp-=zqTAW0#?{X+zid^Ugp`56Ys~KVLM8QoOoP*H!HYyAu zFwjLc?%a@`*fzy)=u=S4osDB1;;rTvn4!hCw$fXRt6Pd_Koiehv^%#KVWnPU26g{= zO~IQ5|HoH$5X$KI5lvL*0Sjf!Lt+gbuPW|v={EKOKsAvTmCe{XBW&*-#M+md_F`>YBs1s* zNWVShU`t+^(u7d{r@tfa6Edj-wTd}Q#kw-#moP``ij0+I_MdAHZ7?vssb*NsItHz# zig`Af^>z37x*jpwGNH;{*fT9Z;OhV1A;JA;{ci^v|LayoQhU!J%h-UaRR@ACAbRc2 zNg7k9(WJe`-X8FY%dJ2N%+~h3Ri|;Nea3io=JCYfjvvn+r4HaXYd59=!SGkVq4jT5LYa=g*wz!Qd z@Av8m2LQN5vEXoH7b)eB75)`Gj^IF}^U1wBI!6k5G5qJttduAn=O|ug+R39*6g={Z z=3ZJEaQOxUO?+UAizwZGW;}a$dSlfkzFrz>UlU!3~TYhqN>eUtAbWyBU#a4Jp>*5&FEc>qpo}v2uuo|f$!yUb`iTP3pO^|qr>2n9*AkBax86f#Lz7-M=F))u zQs>xnA<|Y`)?V>M;Oe>nAg-#@v#IRD40hSZJ+v{C+mHqCYEpEG1h~EYX(dlHIl0n; zq_;`)t_CG6m<2P|v|Qxf2T_1!kpmc1Rj%3g=O9>JEAOY=`jbn-+>-DU%c1B_Q$6FR z1$vh%R<|>`;=pvZ?R4mmp^=QH-k%v;_sj@zZAms5>_Mbi`J67S07Q83sqz_Mh$8$Z zwo8tYbX%E@myoS+ zjT0L8-qIGH(`HEl3ZIf;zD0L)SPeRZk!C|PRRuqLFzgDC<^5MaGgX>U%r4Koz2AsP zIdc;i!TwZx3`DM~BJB3#S@XVTA=bH>Gmj;j1@=;U{Zo@;$pk2zkx@=G`^R575@)@{@ z!#!&e-&@fcTnnYi-pJtzDho@A3P*nQq>V?f`_q;8N4zf@t_@#Za~?mt=X(LyGChek z_r3-`pis(!0nX&sWPsJ*1N{IrLl02g#qvt^sw8jv&!qcMv*0OF zA-6xeakoJ0&tUjm7>h8^-z*I<9POuwk(0K1-5t*@lR7vp80RzxoEi0g;iCXM!oEq{ zp+=mQ*({%`26~2pEsQ1iQVjEk&4Bf@--o}OJEHynmh{(_fi7h zmMsu_)B@I-xqXqnWTl(?{df*K1!(7cA)ma5J-+mB{6dhoOdAY-To(H}v@p|y?XM2T z$Muc(%0cITeV<4-A>{Vu=Lvn@^g#JvqyYgi2T|ono^bWG?!fvQNS!(`wNbPWJ>5x| z`M^CGx5inQhIKyCd5Fh9GMPTz*WPi7?~VmGr^}`$QQg%Wlu!#UDqf}D5A?3vxu4Z3 z4>_El&nUng^EIxOS2lZX_l1UCf>yxv`7tiu%6-jD6e&ZQYJ%qzmZ!fu!*&9g@Jrm{ zymp(63vaOVarsxrPc%ZqJ{a?3Y*iM__}~+^)CaRO-;X)RDZd{LXF0C(>p`l3DH^!| z)}V4}(y?q&CAmtDIc`vgG8$T+e8Dw|CCxm`nCVO@zqKsuF`afbRWAqbXNBoO4Rro)i(0w^<)%LS!9e#@0u+QQZC59=!^(a+s)UHTSc+&fX%m{rI-IKjoU&m9( zb=2S=Z=?GDPmJ*|MxYRwrK)c?NwC2ezIiruM+d3sZcgvetl3Au=z*@I|COfv#oTp` z0N-+2Y5ktUF6_z%+6m-faww>MNgW#ddw^u@klrJ`l`rGUo5R~}PRL(m=|pSQ^c2Zi z{j-r;w_MYjCo++D{Ga{91a&iN}uGkamRpFSA?#;#t<;&T|{}b(Jgc@gd(AxK2&0Xwr?0Cc_FZR7)(Vl;kQ z_aExXLGA8$g-9Bjr4^+AY zf1>6{3q>Wrq~zB^Kl1$QuaEx!q>>Ki^xs@oFuhHgEO%el(?`w{I_&*U@%}kNU*;?P z!n;0VYvoOUjAVLw=4f*QaDEk_ddNy#N0Q~@I9qm1mQK&+<|~KnwFzTwz{S|wdriIG zF@>Z(lAI!8=H4OIp1g`YhZc{3dWQW0a*r^pp#tn{3E?qtOmNyf3b58$wVzWfp;w~fg#zQG8c3?4T}lPoE{cC&6y0t^oPn+NsT-z0sQzAR>U7tVsuVS# z0#^X%g0*EUW4*02@(gmdQ%@+0aTYvJNa~COtc3!)%xkS^!iV3B>e-?6R$>xdNC3aS zhfkfL5TcBO1(F5~_|JD!u3lOgC@uzlKf~HK2UsGuu4w?ADX)=nhAj7@CYx7*w-C!6q&A(F)OuR$4B?K5iCQwZl*NdR!+i03o{B3qT_{Y6!y3UL-AMF&lA8bte+ zwWII$JkV`jijkC+$m{5e9YWxFEEM)^`GHG+S;=(6@)!?GC%$+ch52DQQ>?Tr`xrW88F0 zTYAE?t~)5Rttav9D9R{+bI%7c{{}MXe%BhC(5?L9lJn-~v%2B#oP~rl(U9{(%;r6` z?f~9uHS4ALhoP~U6H8Or#QB-tj?am(D=pk)4D9B`e)U~OiHfy7Btn-*7ZCZfFi^)V zxgDmuAay&S4C?XZJKP;^^$>`!)+yc>>vT$RTx|GMrOrk#|3A8Krz&sa<&V->{E?Bt z%{xv~bke-(iW1q*s~C5=Yyc2$BauT}K7oeGil9nz?sa;sKPLL!wb`Zo882BQj=iLd zgj(0A`!$Yp)cBlQ=6uI6tG0kd)HHZek^9hjl*Jw&J4iMW(*!6=AZ>$*RNi=n zmrR3xMf>$viJ0iPT907TqSvT6-5P;qKD<^kyR&F#Q2|xRUj$A zbP*hc?DiqAL@O+I)l75xb8M0JX|}CO=Fw+) zqm1kY^{OYfU>n=O4K*Gquo4iO2-&9(6BzRR!eQL*1(1d3C+$+6zaRmeKOl{RB|8R-6Ixgwq{tn0vn*nkKXBUOmgMpG}-Nnc_&_vsg@@{vrc=U(JEAGTgHi9^Lg#n!3=tJ+l(W*4-}D!(pD)b zDD^&txe0<9y##p4;?k$h4ILY-!o$r;IuJY6$~)6yT15iJU&wk*2mcq>lH|&th0}~& zcf#V|H|1m9y&G^{@16Rj(mA+Fo1k~nY9FjqY}HCijR-y~O=t@%uVhlW$Gu{Jwljz3 zzOq}24qFg9_x^iqf=>FT`E5vTPEx`p?zX~f3pLfKV$sR(D&(F?&Y>is(-vWXuaU)w zHv>SWM2<}wGe^nfk~-Wu{N-;7G^oFm*-oOqt;53vY#mEe|yX@r90Gf2BFgbD3`_}$cS+F3sAwc9&;VQ02Gxcb_==8SD z2(K)J(cc(z>(8YccNQ&J16z@X6Y!TvozzXm=FmfW*!LY!(62W4C@|)5<;9+ zWAlSPWlN7Oi)81M0Ib6AZ>(a&q%_#dW2i!>s~jIe3at?wSrz;4_*dLmvAG+N%B1WDJ-eqdg!haIM__&7 zPqZIuYam^ptnr2y6V_I3?uh&uEZ=Sf$Gx9$dZ0z3t%aACT_ko>T0j{~D z8_h*3x4&M31^v|~fHqwVF#)jfMqEeqh~L0wT=2l{{(-oD*Vk49--_jF;X6B&=O<5| zlw4k_V^u<|950v-LE2V1K9981zu@=e4>h2z8M|rMLft8;Qm~I<@H8UnW8*gIonzyQnGs*s}q_i8t;dN<;WO^?}rE^V@ ztCw`!Sl9S!F-7A9&wz~dko}Smj>7z5nx2FS7)EDeyy=TzfCL#c;aq<*$%ppGlHO=L$q)xC5>dq=<< z(zk+WE!aD{tJQ^v|8zO7M-`w)gi(zpg|M_}?rpJ;3dy?pK}b^n2xAlE1bE2Y-DtC7 zS$u0q4hJ#wNmAu7EAE(lz3KNvu>&Bd3a2N6q|BWqP*YqXNw1wCUJb)!&H^_q@D>2Z#mMnvQ(xe zm@&3A0&qLD+%e@%xZuvpkOoe^HmEDWL7`x?q{vHTANrSd&7|_gXG@F*YfU(SUW@>e zQbre37rLb|O>8C1UQS=akOUe1&>!!R zIC}Ugfd%W?2rF9)xHRN9XApOp>zh)m_FD_SxjcM88s&J&(QI8(e=^i)#bqt5b&_GS z0BS@_#qet+_I=@0h(VdXS*m$5Dy3!&4o+r&#le^5DpN=bjFsV6g^yRuSq;9(mc<*x zx!oSOmmH%5)0)k*ElV0UwR8c%N}*(_;MLddWGjd;Wz0=a8&c?r-~edbImqs`7y!y$ zH+9()k6Q)2fPxa71oJy~KB$!}&fTzfpC}Q1>smpbdl#boRsw^@-$GbJ3jGo}SwMj_ zL#da%n3#B0{cnc7cEo9Y+{IZF1XbKDVDLo=sbBH0KR`n8JxWh0Gs>z^1IPJ@RhIb1 z1em6^5x0V6AHmvj?3Vy&F_wG%?P|tPj?`;Vncan|vD{hHN$@P?O@Z+wxY4TL_aNie zZr{dD6x2Snce`_rFJ`l>-X2Bu7indB04C);>_sSIuu2_kjkrDe%T>ZmW6inyEZVkx z5QeyK)ObICOe>S5YRL5u)w4YxFmm;9tM*=9_`tACTZw2tJbn=(6#7ys#g(-4aXl2M z6;t~p7k*`d;KF1uo54QD>EalK)C<*{l=2^ZA$x#Z?s+d+S&}wQxYqY1z&~CYwcg;g zy_OaCoj!*9LYyMN+U^ENp~kUB;4JwIBvp^l9_G>DownJ;@@CvnGYSKc=T`lv`kM_D z{G$?Ippj9m3LFLSTen^S_lsS9>ZcJ=nLCUUAUNWl z+)w}Cck?D~l|w_29fJ-4hkLBs!MCplF~WxtEIJlo)*=VzJlg1G-#Ov2Na3=|j1c7f z@-XJk`i2@EeY1GsUZkK7XKx823_1I7ww@0~N~IvnQ(~E1jKxS6R+j1TR;IqSF}DDW z;6wqNmEJ2LO=KUw-IbLcQ{bj_B=hZUuuQc9-hG7j5!FFhj^bbMh+T;F&mr2bf)k9f zCEKH&U31l{X6)W0AM%_ajB`E9nLEK(+ArAKdwF4*g5hi(RS}OJ_39Gph$in6*&jse zNJyT1<(OdD)%Q@GvK1Gl&s#wAur+o5^1YEoQZy}_A;)v^BaxddY4t4H%_wFDd+UHQZ$;^qwmiX zsm_L=C4B4a>GeyLDW0>))$jhoTNxU)?H6~yzm-98UMW8_3m&V``U4ai^^mDe{jw8M zw1)i@k@_d4G@bHLiy-v<-E3EI5I4n>>aAS4o374UdRZ|9=~X9_In*>*P@I3;`9B~| zr%+bL)id*?Av9>Jy$KlXBa%`R-~!FUIW+HddFbn(48R>qq_Dl#WVSBD+;r$@DVs={ z3=D6xx6FFU>7vy(M|)?bK(I>R3ehB@^g$s6SDBlLig63GWG{v!c2)e;gv+dYWCJd~ z8MQ+l`c&-rnwnqM6aIGoLmVSV*zQ6=YW>wv0NKvmqCTe1TzE2(y~$WvqDynH!$VDk zTE>{zoP01ODSxcZJhuw1i^q?fTdN!>R%#f2y_({x|bwMGLwUfO}V((ach9Los(&8 zJ=aDZl0K&=&ffj)flG?B)wwyp;-VHfhg4aOI1;Z?ztL=Z*;C_VM2MrM1rVAh4u^b` z5i3+SN7%a~Bk+bqAklB3fQO*gU_d6Uh3RQF;O4!XMeSJXu<-Yv_Jhk05g#6!kFD`z zG(P?t=7om@@M)J`zI-A04`7LABL%z7%T}g zL!uLEO`2xk8VnJ+`(mIN=+GiG6X`no)fC)}o9i?XW(N@iUCWFip|kJnzHS3fCnYNY z0tE4g39#NZzoNkLJ(FAIwvRt0KWj3G**C#3zI}zSy^qL02eIza0G}W4fA+%w*cJgG z>n!`!z#d%yfc7`{KRW^gQiM!z#=R?|dxd_|Yo#l}=c2|`#szGl&{(1etzOU-bRWC= zC=0XzA(9G0`+p_w9%KUV$?gA+{T_Sm0N4MMtsw`${9pa4z404;1M`{AMmBoY#{}pN zyL-*&QElXnuW zRF~fWz$F3KTaRWro$o9^$3adPnjCai5iK+SR9E9H5fC3JR@lMp0jjq`i&rCX7e%!k z=L^BsFpVUuK`YeWKRKD=2yjUr(derZ!D=yU&3#oc&WbRfyGf7996?t!@3xcK3#cuB zLCx(Hhipf~3$FcQZr&jaTiHQ)2R|D@HZ$_Y$@qKI6%KbF)g0gicfWL{$4H21k31)v znZ@_GPQCoO9AeJ4Fx*9KT)AE9aVb5sb*IA$@iD@0TS+WU=o|nUiWRkneSkIYEDbi- z#Y{|1691Kyt9PR<^v+?5z@L~35p@W6LX_x*C&N_49ioDS#=1bjJPOsQLP~oh2j&MvcmyNsz}+b%wvlW&w1ADI`81_r*CklktBsuvT`LP znDytEM=;~sAm47LSZo)}#3ooEfK0Vt_^?|MFe9z|F?+meH!V)$Atx(K^QwB8o85f{ z7W4Qp$@OQxy!j^6e4EaX3xZ>|YM*?8Mbd{RG)orGmze3E!y@`hW0>3qlPFdP3kwJv zY=E#~Dg=3HTTjOj{OA#vi6??Lp>*eK;(htW^YUPLh+(O@3QfYiG5%zpo|&B08;573 z{)_IfSFjiD-xTLdU@mT}2kr*kI(FV5nzWcAyPV&w1 zeLXCBS)d}i>d9CEmF90-mCatRX5Es=elsmkSF?bFNE=P&Cf+Qty&!Z5nBNityx!~T zkR;C(vss`EFD=W5u^Ii7V8wqAfSwhk{Elr9dmK?SE&Dx^P%-}*6~3>XfTf%o%?vWp zcP4rHjSWpFwMxG^Us$e@z>kr4t&-B8Vyx7kLyu!O!u2#B+d^t1_Z_bL_wF z|DCG9Xq;9uI8l&E1#&E|jhq?V|lmw>e|{YdiRh)dEXd~*}|D=dUa8DxtOf@?&utMaze ztQ<}21}}Ns#e6r~E{8RRBI2JSLTY_z9{zXWZ^_5phUeKenSYIK$FKSSUo{`EeZ-l~ z&Ybkh%WlrJ)g?d=_x0R5+{^+<)W5lq|JHpy2d3%Y9SCN)p7FO57o4m)bGR9~j%7kf zP=1wbtL>vme)F z5%{JiFQ~W9xro}5;_iEDEb%GW=wb_x;Zzud)*QA6jg`pqj_X^g=40%5kS`&bE$vaQ zS#LF$7~lWRe*>xkz+hq4m2sv9y1F;f2$RUP5s%OiB5f;JUz6)qab;1&#x*bbu;O{Ac=G9#6D?hfDK?)O>j;G z54kRyBKOe#p^@fqf9?3fqbwR&& zDA^txbQ6g=0Vw^7N8(S_@a?UHtQ&J}rshEwQ_F3sf zg;6{%9txOLbvmeo^BX5ZB7Ra#xyzZTLJJJZ)hG7_oNGu8fpF*U-iB)d2zFLr^3*KAV-E^IG_5fpwsykLgi$aFsZOH1a8tE{q{454!la7`$<$t!j>R#kk zw+}A1IC#n_6an<_rlR%yb?Zs+xL7|Gc96Pg4-gRNSwSf>b`Rxw>d`mF3cpO{j_|)5 zDrFXeY)L2ZfFiq91DrdY{$j}U7-4oPT(f*@$AC32mreh1H&1i=eWjnGgz4mbpD6&~ z6(gSGNd4?Q>n8jDnn@i5EPh;Z zPGdhCM`B!0bn(mT7Htu>Lb5|MPGgY^Byg^uXBu4q=RYr%avgSbvt(5x{b?S1n#iyu zKL)MdwdX=ebx18$ijkx?8TxAvjaNK3Y3-<(u0nild%H_}olvdE;EOp|Cu<~*J+bR< zP`urVO0{O3+I0mtb5i+8$uUI$080ec00w`u3HQ}Xe)5;*nSY05>qu6+vk0!>0Kg|j z9P?=YI=}W3f9I*xYwd<#iqlB_0Kiln?^@;M0_+^(blKAy7|_xLSbvrxvHy;@N_jEn zvH>)qpO~23vYZHaiPTWhXvU{%j-sxGk9ZwVblt}(^l&P?UpY}5gxp&q=4bKJ<+}c} zFCL6*#mE1NIjQa`q+3M@(PwLCPpa>Y_t~P>G@;hF{dqu|btw#7|3;2_p!<&n3QAIk zUQQH5t^nu&vi}|Saagd;yaUYrYZxqWz6vqSE_KZCMs7vW39W@I+{DoAG7h8u6p2Yk zhKG(#?BiTMnXdcDj)Y3V(s?hNfnp`QjLAJuhn4OX&3W$i^kj!6DhDUTKvR@_{0VRS z4!K8+_1u%RS;X%tmu(JH^Xswge&&;=WEe3VxmJ_%f60L7+s}Xpj5PNIm?0SX;OCuK z*0wj~U3?Nl_t|dYjG5}ek?Tz#!(USKL$L6x3rFHZ-NJSi#WR9fMO~_F#L8+Uqw99R zHoK79PjGm~ZdEK%LIO2io6cCaQ$C zmSc+^bPBQDPew^+?;``~A4My?D_ReT;cP<2BOU>7LeeAn)*ggRT^{{bRn!USZazNV z#}}J$2V^0W1>r(!H)PQNn0RHXdU;mIVoN`Hk0NB;Al|7Z9^KLj%xXH!K^0PtT>i|kUp z!=<;ofwy3F%DW`cCFf+y3$qJ6QrpVgs#YVEI!edp=YwSgjKz;&s&RoSWIJ#b1clBK ziI0B-Zu3B|S4XE!^{cKu_-obVhQcEJ<-Q(Mj^Pcqh4p+fxRPP-smJ@il!p6RC3423 zne)l;05F@P4lrcfZ+7ctRV#NYv-y8=x;t^kch#NOUzPX5eqVGT1)k``;c_sT-JadU zq3kLHra@=|TsTjqDeow66LMcNio1tCU>aubS!0gJS+kdJiJNaD>Lr{{Ft?(P~7 z3c>K!>2vH#;NcHDhR9{>aO@goSM$JuOEm%{!Jq+Wtj7kDFg80q_+^ErD=E_|71^`0 z5+Y(Ep<;)PUZoBQMfUZzMfsmMx1TV(+e*qz&x@M)t|*KYC1QO=(s$TJV;kdxX8iYT z{<%3LQ|trDqO~U)Bx}?6LJVljw}bAXQ3;CR!?0AMt>AKT6W6{5Ze3&j_B_ZMDf zW!1vYPwG26Uc4>x(&>*=4X2JDy}k11H2*v1pK$u*zpR;m-fukh@%__t$1O{4xADL0 z)b_N!@bt~Y4TbAV2I3QAHa`;UPgg@^2>~ZLnv&!{d5*tsw<8nO_Rrk}XxIKzpop%! z+Wqi9=F5iQ3hdvV^u1!>^Gbfbxu}s zia0LKiFO`BVlR6oB?sH;;a<;{&`K?he1uKbg|@l~IzwEXx;UIOA>9gIMD%Bn=vQvbqhv_BEF%c?vLSc{8rYOq;9_TtBVV>%pL{?y79d_u z=y%8~N>}`_;JYTBVuBVjumjFPFFLV+&h_bEOBvzLx22Vx6fTMNM1K6 zLXY})O?!B_`&B8}AOAwk8q`r3_yS8GPt8zJcDwF#{#b>(I_~xjr&cCqltRzr%Dif0 zBJLE3Z*5p@S!txqe{OR-N!H)dz-zi+d%I7YQf7eTE*K`xT*}>u$Bmi{&#%9X`)oJ; zyyW1p7%thpmtz7k_!}C!EGDImRah3SHyTF<(|2~uUvqFfy4wob|#Pqx}0s;sd8&-*{0vNWH@3a%i#v69J$*e z;5V>^#bf?TgSBKCm+NQcKP>BUv^qs)FD;C?n@iiA?qRP%s|cMMw+nyiqTsH4r6J*D zH{Vob4d`$dGJz^qXX#%fPY6B3W{qBNx(;W0joeVXYjpJfLrplWi{0p+6c^_FTI>}& z4xd--neR*8YvJVuV{HQ;gC(vx2te8s#GDs>ojd0ExjMq)#}K#;LZY19sW+m-rDvW6 zz!FaDr6(k{b?(j*>y+juo5N{TB$Bpn6-w}bRrwbA@n`;MzS_w+yR7(?TCbCq0hjA> ziD}y2ro!WVol=fkb~k0to%|&+2YtmsLi5|Nttn*18fB&}mvcdj_@Sdy>Gk(^*K%y= z((Cbli=R)&Ae!HlKUCmfWqggCBFtz-@*h+5Emstc?`6B-owH^vdRhe+p?>Yu!uc3@ zc#7A#zSo6w9?kjzKeN4G75cd{fq+y7!P*)+Txiiyn2^CaxsF@qZ$KTFa}qj+^K%yC ziSlzpg{hehep27(ojM1wX5&*ey&89~1C4u`E7w;ThV0x+5(82lpTm>2` z3hBT&{aBBo#2l0rd;SDCU--`WvXuO-Q6nmcOn`UCSUv6zZ};g@DpQEHR|4%x!3_FC z0;6T?!p~AcpLyV*aL+;5?UaeLQ>h^;`b%tcCvqO&*p#byQ{2ow(~XQab()zi96dNA z{7{M7e#P0?(78m$aNh$8AlJ@+j_0&(r`X;6pp(*(UmKKXRqg8fJ1B|p?9-PhW=i%O zM~WJVb9Qol8(rH-4K3wYnR>;De_&Klll0GV9C}1W2QgNjeS_rikuDq8-|lCu{3;=y zQz}PBzn^cFS8GvktI&!Bc;x26+u#eLZa}uUomoX*kAtH$!Kl9V0DT z#a5Cvv+O1Qq*{dZ&2_5_%?alAHm-sSH|&pdhO6S#lX5xxy3BOrZ+(Of(OMZUBa3#rK3`ru7;L=uW{FO7oaU)tt9u z?~#t+gm#n$RZz~mJz+4uU$wTNX!qyhBdJoBzY5-KQtQXT`d3jU7Xv!V+DH9wa7qYD zdeV9yk^1a&#oe;Q<3jp@;_@i{9*2sXY$F==E=Fu>X5pjB^X>vD4|Q>2-c!XqCy{OW z+ZMAsEoXQoO1z`mYn|&0A3aKHpUrj<`i3%7*Dsih4)guJYr#glJY#t+E5_~i(wr8% zH)u2>W84=pZZA*(o`qWYkD5xWm*SeNuhgG0_t0anFXqC9%AIYC=l|DaNRJ5O)gF4UHw3|1~_dcRJ9+;?iVDjBFx0Klci++`qmS zsEb%)=kKH(p44^^DAs$WEN*!XF$SL;@JFzJ32ExRW8_nq90ON3lY&6yD)C@qnr@HL zAC}WzIz`0pi;W#+YxTHZV^CP{V~p+0O+w{hmvb*HtNkN!)kZQOE%Niu7h`iaHN7+K zVF;auf@+mGZp|}#Dr_5FeyEp41=+!hSB(mB$9@O#-H(iDi^)sSy%d`V`Pe=?k~^2S zJycGb$l2Bl^xfXxZVfCi7i2i6?p&RrUYhTdW$C7sXk9Ab4qq*@{eH^UyYtY&`H}kv z#**&qI9OgL=<76s1>^FWGM93@%XG8{f$RH+7gT)gFJ$wTQy?U)_w567Y7w+%Zy=>L{6Gw_5{pz!aK5JqmEV47WmC`VsF zEe;RVO2;kb@A#G9X!^CJ&x{Ls11I0^d)!BwfjVyrw+_RR=aYLy3l1)mUmqSh@_*o; z4erM1He0Wc!ZW>YB}>@(zVO{@Tz`N6JSiij@+;jQhsjZS+!Bj6W& zz6cD`i6V4Twvve!>*gd>#x~#-$jfUjLCIRY#abM_yHbg1P8QP%?0TH#Jey{WwNWaF zoNsm4=lM2jF&sI3H^HpDI>x0{HOY^V{F5Gi!;#ik(XTyaQ<^8OAjr`xRy{8}?6gXc z^$onrYm}6|DELO?4k770W38*_swB+$Lyeg99jn9xNgCC}dljn%F6fK-LgpY9Orl2ANWu;-OxG84-&7E)cnqir-%%KutA- z_4RrNGm-d=sFPsMTh#ml{BGgy_5-)CnK=oue_k#hC!(FbH~l4_Y2Jzb7*aMB!Rh2t zXA&nT<26pqUs}pU4x5fpsp>jsyxY!bN2aYz*;pm^-|vo!j=>jV=a+`Yi7!Ug!D~Js z+)oT*gPpkgz^q}tGPRm-dlZL~05m6!Zog>QDNylQpkVc=#yD7{fUK)p8JZp9VzT5x z_XnnE#8zExa92FJw|p>0?n0jO#R6rnZ~iVskI^T>dPd()4&Q{q*b5*!ISx!91HZuS z&jPM9SjPe^Ps_z{DbF`0|D(C{jB0Az_PAc_Q7I}a7>XX62NhJL#(;{VND-9|Az}cf zi;z$PM3G{lDAKFaA_Rz(fCNbdlok*WLZlNqp(7z7A>nPn^RDNP_ul6>Mn=ZYhn=zZ zT653&|INAgn(BPeUB}IIR@zp_tFJ)G0^^GW2L`5CSr(4(QA$A#BP@zzUhKETlHQ#u z>bOlsj9w8dlt>HplE|kFR9d}3m3+IOwoHKVntBD7oBMvELv6KT3kH?{od#%_;; z$Nj$U*$qP@0PT_0HFft~2j+5PjcIYI*_2GU2e{%marC2t|(lr+~LQZ#uBd%rT@rp|4#S1FY|% z-V*ukFb4tZNNWDF2#B}8#?YOoy{_%>^Mtj~Vvt9ZQvcer8ep7ulkI)`cjx=5f@Keptwp|{=DpTz-ZuBNpDvhN^ zp-qK_L*Gb_zU-RVB42W0wfCqtQJVx^)YvO`PGG3k>^m&~ptksse@OsK9e#vrn<&Mf z6w01nd$!-lMQ>?j?b&M;N^0g{?}-hcGcQQ|b3f)jJ;i3C`;4bcbYNDo>`D}KExV3m zVl**GZ?F&_v{v?gt@w3yo>5{&t%xUhVMa>{N&I)=8y8b_sz6V_jv&Rd$>vea*t9pbl!$2Nu#zd)6pyJ#Whk`P1^0gK-aYnQS~>a(e- z7~5xDd$0?6r%e0!M`FthyFUDPycWm4`vNoy)wMXwf$xL$S)Zw3G+Ftj2K4OKXmPcG zFu1$)L?L*rI(gk8((kMtn|)!*0rx$YlSrhcQFIF4doi+`;*Vwmwx@$-s(guaX-bmXf2`qxE2Y_440csskrYoH?9q| z$pIcG0s`7eZ#YYRc4zU$fv-(MDXyPxC-#i|DMTCtwJCK5zVfUgo7T{#?&cv4wcma` z?;T?u7~Fo>gnHD4wW8_IrC#%2W!;ikV;Bzv22Z~g3{)b=&) z-Q0>lfp?Z+|F||wmO8X?yjkY?fhL&*0YfdClbZdVSRI&Ge-xR#ypxn3dbw|k?kUsb zFsg>6PJU*o{W!FeX+{~3$X~A4XC4_AtfmIhochD!V^k{O%N8_FNM6ra0A?D>>FaV@ zi?5~@=2rx+iL^e)Vbx|Q?=*0FA9zxbR^d1H8jy$+@;PoCaCK+@UPA@&b@3a2@Jh<0l@@_@>JIk_v#?;T&6 zBX@$jXU0&ptcG(0RN8&v zQnPc|n^MI?yYd(b<7!_x6Nm+Fc*z^5b5~7Hg9TUK24!tSdb)1zF7FKlv_Rl~BR2|Y zpL6n2P|bNCCFS(!S`k6tKiAk3THn00aI|zfI>1K84_Yds*O$~#IYRb25e>BUMmP4; zU66JZSiwp?-%P}DKg2HIC9fz2d7LJl|Fm0&PG zIBiupv}GS+Kpx#Vg5sS8!S;~V2JIXy#zV$b2 z+IgU~?_vVB9T-q)uT=b{3V|S;Zg;Zn*q!dn88JNpP1gj;sc z{fHT~CjW;gG&?lRr>L(5d%%yZ$!|TSj2YOi99Zrw{o|o?)%BHh(s{L%su5l_HSH)D zPA+xnTGl+8zu1kv(LL61Z1*4)CSRGsE9f2555?`8=Y@#FPN9k^d$XIiAE%z#4cp5W z1^>h!YVw1MpP2Q|Gnts1vZ4gqvBsvynd^3E@qOR~_@m4bvCj};i{?ib;p{duLi zjL_X)e(Uc@`gKn>V&`yAV^HXlaVZdde4EEI{4*ZM$b1?tO-1W$>K{Q5-8=Jjg&15nOYpggT*Bhl`Qdl<6z~qxYDPT=a(Gv=& z(1wdc$~QZde^wot9|i4*iE|SwDdxF;Xtr@X4Spy%hs$iZ{`G~#Bg(;0tJ%BknkFAf zx{?fCnd@#1@MrgoS^f{0eyrP{!n}9%JdHK4uUq>8&80$LDMjnct3E?LL>A+N%}0h^ zKfD_q@-M}CNN#k@{tWb)jM=Vx(&Law2DRo|U4f6HW${KLj)ecHMbRT^Tg%@Q2>=?_ z`c(R%r~N10DKj0}HlcW$d1fIEo7rF(8SvK1EU#Q;Iu>Em!T2%LARhV+u>qTJB&_$k zqRj}TMbe4;wlfo)o*T5OFEPWub>0@Kw35y{11Y?&#ScDpRiATRklj_zv+YIF9wU%@ zLlZgBy*tj}roD+|o@;Kijf23tsFRfx38;_3HVQ{zS_&UF4Ui;9xh=uY1}gfN$a(Ye zdPn1@>KIvV)NE+B8hEde=^PIHe(J)e4g}RBYmY|M(FIVsW|YB1z-?xb&xro;FF~TS zAXaZYfZo~BnG41J?T}hU_2T0dkF*@{N0`M(XYMOKGs9VWqsz_q}q`5EVv- z1uLh1!*h%F<3a4?r+>hpl*WZQ7nT`8YYDKC)t(a4+6pm8+=bZ8b#*zKc~vZg!PZs< z&&-8%i!P7NGq&^(7@9vejkJQ^^pN1&q~h^IQ@NEE3XiVA@!IH`B>S&vcRJykyma+U z&I2xQc04iJeK+&%{U3^POBHXy-~jd<@mtDzQ}r!d)?L9^L@J})al%jD=t4NuRqYmu z+NqV5Sv;8hNxl&Q>2}B80rZG@r2-%l%@hL_K4XH5W-TBtMIBCqG4sYz)1?wcW1b-1 z|A9TdyokHynsHy%TltGa>;o4y$uY^Pw0N&R?ui#28?QFQp;FWC|cbFWXi1qi#of^%~+fdviM?a3YbIf6|r&4w{gS0hHN!Lkp&xX=Pa znl;47<}gR2zWt=t5v}fcHIGlqg-Y{}X;@T{I07(%A@O~hHPs}=J%l`q+E8 zCawa4B-2XOPk(&Oh?I^8YU2|qM$54W1+G4a9~^CkeX+@l0v0(AyTsq<|j2+qd%zzr$k8pZ28G2(+14tW! z0@&pJcH}%lBAezk!7}X7?D1=RdzeUtW_Ix!pQ9U+b%>C&|n|srQ*57)=GX;KTAQJD;m0(uHHEpm{t z3u4?IE1qiy9V=FEAd22+Jn0oHg_JXTrq|7RJ!_MabaH~=m~&1K9?9$(czP7uaNPux zQ}N>6%U0FDi8}QYTd3}w39xl)*v$=uUK9m>hWZH<1k7IVj6>O-6R|rdu>{^&hKGXd zgV|e6{T+`7r2#xnbV2%N%ir6E37nl?Fd)4CA0PqP9+-!DN}L?Q<-DC|k8c|I@jQZ}9GUjCE>)=c z?sA->Zzm>7;zqQwJM#M*+hxjt7U-nrF*1mN@6Kg}7S+M|Qr_hmXaN-v%i94$XSXfK z*WLh<>%UOe{a|pUa~}Vqc<+enW_@Yb)A>U`ic~^z-<=gLTZp`j-P~Fnff>eFf82L}flE>!gfqQQGNBh0#idw`$O6 zeUWXZHRFrqwUg&N!jrp{+CHqip!4}};{Lmc2Fj8X+HEQAm*Rd5Aj?db)P0ODRId)D zb0`S+Jk?Qw+NaZR#9=JHOnpwUm5%@lRodfwe>AsnrfJJWsISyc|^J^IUXV#*Y3a~!L@o`NyN@R6-x*5 zrAz)X^9Y@!Ix#3}j)ss`FZY^m^mOn|jD>{Gq?1UEX3fInGxw^glA_pPAV>W=Nox8+ zb)m(pWz@rVp!&7$b47ukHvcZ0O{*!u9?HTY_{K}Z>{*|MmsxOXpeD)rM^WWPo6=6G z-m|0Z@4j6h>_SP#75{|2CppE=CbC_hX4(B9RS_&$*l~i30Xt9hCQbVF+?*__5L;r# ziD1u^9dWKl*nf-YIU7OotjYI#5P}Ah$m#u*q;nbcA-Zavx;9D!5b`IXA!`E^h__UCRksE zL|A`AE#VHK598ZA?F6SWB~0+doPwgsac-$*MrGz|FV{N^uEYl$jtHDhJ%YI+J`AkO z3gqL$@9VNz%ZknE3(GN8o!zNu>NfpeA-bTcadDnu*O~XO@q1zs?tkU1-3v}!HrZLL zIQeE#R-C@WW$Mk|M}3t&m(b{h!o`mnou1k#kB?{U>r_pC1F9?Eulxck8Sn;BZCe)p zpQh#Gm{W)S@!O<3i}nlW?Lv8Lp5<98!?KqJd$nW5y!nw&H2yMSl*6%*M*@;TLbR!* zK*4akMClZJKvHQso#1v?wSd?ha#)O1xJ#c8kaT(;&!f!M3wsM{icJoN%o=?yCU{6P57ZgoHp^_SYG3QH~@2ZV0)o%ihL`RdZWHnXg)Yb^U zR)tR^YKxcth&xT9^%}Wz+SOaM6_KW1wT{*gf-e#^6kWq&ZleWU{gO8_fRCSEV!a3g zlFR6E+7$U<_}`Tz!S2?KP6J zXW=`H^pT%!k;dCAM*VB;$+^L`5rwzu|6xA{pTN2SEO-|LJ`UVE3c*8LE^6+U~F`+UT zr|H5S^;aVi*jnf=u-@eZWj^Tzqb*~KwoS4J0A1eI?K2!hTwCNqS_>1yjPh7jP7w2X z_M;*9aEk4UOrXI4EGm^^?$1+X)z`}10Opr3qIeVOLVsux@GUC5Oco`|*7A~SUv&Ix|I=6aQ&czagC}})*ESJzFq=rY%~E$C z{n&qYcJlaKb9^qN!wZ9s%{^wj9)=hiD$odP5b;8FLb-Bdu9R5_hsB*)j^$mVbZ3tYWXedC?QX$TB@HIAeKE0Jc`%+yC$xk8cy~&L&IV%)Etu! zM`nTVt*vIakn}m6w7DdD+gQ(q=>)dsEIF|TvvMtpMRtFju6j#ooeTy@czZS9M;tuAa-x==h2Mn| zC;)$p`u7YOJ&X;;?Q)$-#GPbYa}{?dVl7ji#PZj54j|5$G@M=kfPYy75MeT_Zw28@ zf0Ip46)`e&q)P3Gk^o_5Vw?@^1w+Cr!1Q+PYp6DP2jwH1ie3_1~x1 z8j()|l3%~1^5ABb#DB9TbtY*2BvbxhD)eD85KZ>{svB)#Q8t(p2!dNaAO8P)xT*=~ WSh`dh9#&c(sm@h{D+QPD2L2cS71Z(o literal 0 HcmV?d00001 diff --git a/tests/milvus_benchmark/assets/milvus-nightly-performance-new-jenkins.png b/tests/milvus_benchmark/assets/milvus-nightly-performance-new-jenkins.png new file mode 100644 index 0000000000000000000000000000000000000000..f03b24e647ee5c863aadcf4abf78e0ffd89712a9 GIT binary patch literal 45034 zcmeFZcUY5I_cn^%0TmfUlq#s8qM!nTv?wAdEdojefmo>m(tDyJ(xmqqq>7YC2Pqj) znh2pIHA3h}OMnp4&JH^7_|9);-uI91ob!F>I=Q?qjd_x1uf5k^`(F3D*W-UfTWv2Z z4=W1`%iinPRCHNbb_=txY;oGX6a0^iYriK8%Mq6ADwl7%CVm^vD=Zh*& z@SQrf&*#ePCnpZ<^EoVeEBCggmGc)F(;b$2CAaJhIDG;G0s;dn^@nzaFF8Mb;b>1H z>2VQQB$AM#;10jPp1!2Ycm4byEOdbbU?Bf`J#Dz*_pdkLA3Rrtw*K_q ze2>rWpWddNJ0N@l#^Wou&}-P%z_XX{qISXF$=3um4^@r5NwZi8?xmq>h*BPnXlu&I zLo_Zu8`@U!%r z1AAQ!7lH{PcUsZ<*AL~T!m-B{+wzq{{_Wlot3JEcx6PAkQpd5K46;R@&K1t(W*blG zFoY#S7qdV{E|VC>B1pqChhn2+6tCr*6fE5lQafmV&huUB)}JS|ot2d%UrL5U&`QrO z71twTvu3oI?Lh~YjlZ7so}J@}(_?ioxq3htR?Ne7RWL9q`Y0>MA2X3+eL&dA z!ctfXl6MM)f90Z%7(6))gRwl})AA+-+Z68~9VS9c37;T~#*85Qq~5eoQ$g%A!y%ub zycID~oRh4b5xxI5TXQE=$~!fMl=GjQ%hgPQ3TURlZtMGn`m*h9fk4=7f};fTO>7Hl zNbZ5RE0@V;JLCEFPG;@j^2dlYQ}&OlANAR7kka2Hwu}0WH2hdhrRllFgqd&4H3R$h zA*b;@3VF|0Ce3yR4ic~L%Rt3_W8}WQ^t0m~x-3)|CBn+liSOA?Nopoi zVMZxqq$254$VbREv;XD~Lu8F1c!$;nF`i$fu%_NdE1>BzslY6n@(UG;o!fat`mL2Q z(T6qBoX})aQIM!1=DqEeDYz9w2FoxJQsxntXe5;u6lVGCj(dJ}nRQPYb>L+(xA1tg z4qKUFx6TNi=FMnp_}f?SYwNSUa@Vw=kkVk~7h)xB9AkM^1fL0;1IQVp#dC&!OPQZB zUeZ^@PeP$`^~kzdh@12kfxG|*{xe+B&|<95ZnyNF2;uFGb}NF6f|0$8g7jy!e$;>U zCt-j=zR)XvLGhPHYt?7{Vf^cXH{5w$zq>RUC1 zkWVj;v!C^)ZkY^){KiMv6?=uw!bv#R?(UiFvNzXVYn#t*n5EXosj%pYW?Qq=D=ow( zuTv`J28wlqIfkWncfOpfp~*6r7Yi2z5G?aPt0^OKN<7`b~!>!Mpp&+ccY@ zken*^N!E?R=N{XDlk1uZE3r-Xh*Gl$t6D8DeuTAgkcoi%g84TpWJnoTEftkA{wjwCmsJ?O18Iw}cs?DzoGM0Vls z+KBp25Mf+-`R#sR<{wM)`G*fQbdn*g96WJ;Atl)g)r%Iq$R~382ZgVxGY^5rmEwK4 z#|KWTuq<8p+qr+0K|~%9mO9IUQWFdM1d$vOBab^iKBE$vt`LneHXqv;Nb^n3Y)N`P z?tMwukL9biX76_IL02DMCCjqci9H7`wEftbJlwZ`ROCKa2BUMiS<*gEPo!`?pO0s$r{g zN%W5(^CycMJE=N;EROyeGQugGe}8L>&0B9S_b>k=ZvFm~1yaBbS&Z+`1c6BLJ*55q z(NCki9S__+QY`2Szl+rwvj;&tgZZ+)>?Ga@5%*Ovw4A<)q?0dnpSRo?+n`bt`Z3w? zqha5j7|*V#6(jF6Q)w1DS8SJqq;#lPH#vpas4zjujIF|mC<%FXX5 zzB0wPOOo7=V#M={+K#VTChVAhn7Y?3ed66ju8Tnk!B@~ARqsi=O?CI2?YB~v{{k6DdzB1ejIO)3 zJ74Vj$ZvUrXU?usGH>m5I5%BpvH(4A|908@h9{Qql~nSX61HD!VoH3dCAlUD!V{B$ zspbn8XZwzIA*Dm8C2zy1jwv*iD@`~G&tb@Qu7&7$_l#0Asouw4?K2r6XR9GR_blkS z5-RI?u_<#tyPeuF{xsgO4a4_Xy3GUYI+=x+%i8!kY|8%Ec6~m(gDY=&F7DDgDEzR- z6jCppwmM^fM$d1_TlvLXfeH5TNJ|JJ@s`M4_m^Na{?u{x;6M#t#v68*yT{5XO{gte zQ=&chG@q^ zlt?zgb7rR+9Vz~*VHg?dvxsn1anc?EIO9XyK4{MJp2d*K&4o8SqjJciXQ{r2!2}a9 zxuXEKN_cTxbKJzOQVW^;YY$sEq1zko3Mp5uBHFBVX~J>!0jblbZ0tl00oA~?I(dkL zIz9EWF)PPg_ejXK9*M12Dk9Qx3}|;qaUpjBX+dtLhh!|*ynf)p4f9`2mSJ3 zkLHMZ-6b2rP)x6M5~`bV*hoCjMos<;C-{Q&0EU{rI;~K1w&Q-an7zl4#l@3(3$li@ zBSilWMVTKWd*}@>M`+fjmba-9lb-A49t$3~&8#0vtVEneK5|D&ePV9G z8_9lxjVR*aAf(7FH0r2=BT`1F!`s;^P5(RhxjK1%i`gA z;b#r6Vv;r76Nk}D>0s^4z}oL5el4}{y(y%8&~jO_S;?rznSUy)XTrr^X1NU8uJB{d zbn|7GxRN_tN)mQSirn1@0G?oqAS`tvWWBT3pvly6hADd}#u-f1>Gc_+cJVOOVi$$! zFWMAtt1{QM*%vqIsfWf_OP5;Lp)uTJ3nU zm)2(^gv@)JB$zfY%A-afx#>VXHexiXy0hy;>l|`yxO_d=BkD;eNv-H~E9zg#1u82M zEee~{Y&)K@pAmNrB8`j7^Pt12G-OOON?)`vYvK`mcn<6|RliAx-l;8A*B6E6vUL-G zA2*V3bnnf6p?(xmrgx$KgRbA*3gGr69l!Pm8B)x@!T>WzaZXS(U;iOfhms_wwKTyP;u zbR)rP8udNTF;aHPzm}Z6VSJ?8y|u0gc4s_5zdl?xtzNrD#{v#O0jx&9f%4V9Is|P- zgP%}o;)D;@t1OV+5512UC$VIiWIrj$k%4N51Z)K*A0Mjg@Ne$6cWtc0WcCrp@9=9lH0#Ck_a&u#+Y8 zJLnJGUg1UXXfELwM54>5>{v0{c|zYAC{fQvIErpH6^hY!s@XppLruFkjxKf4-`<}* zwOcKRdjr{r-gO=XI5Xx6Q{LIF;_0SLgi7PsdQ@g=-fw-1H|MExaZK~sy_k$FJoA=c z@YdrzwJ0nvq?~gy>%RYCGg(+@^a$jtP#x?h8%Oy49na_X;Uz%+Ri1CMYr2Sw8uzvY>JWJ*&fpwCF&lR8DtjEx`mK( z=G|N3%J07NXDEIWTu9shy0HmmCoz>JboOfa!Z!F*=9Jwm$Sk3MxqQ0H?EKNIbRlr5ygWp>$)h^sVtwo0J4 z*-7Je$Q_R>l+qcum8zr;R=YfnX60yeOUQM4^=pPf<}o=-$ukQ_BW$aB$L@yvN$-2~5~Mx*WvuhO?2 z!Po0wCVXWarS&Z^qfHb_4b^W-@i%4eNy$bb4%hH8o zF)R&` zrX~XAi|V(h`0oT$GXkb&l84Yq?IV~e357(*+~-5D3|Mth@NB1B1wjfL57?t}CuJ>lQKeA(`7j&o!_4Db-0+#)XA@$6I^l zYL?wEcVD8*jg?TxI}D4+h8_^zMAJ{lCk)=oDwr#ZRrflb?GdUNiTNJCdYZ8xuu|=a zR3WfG`x>DKVVu;8)T%O2P zr0iUkd+-S{xMs}PF|o?T?}ldupCv~X?iWPPexWmMjGl}j)2O;cd!gr9zV$CNj&J8h zUpvh${CVGiM1X{#zBAn;5;CMFq#P0k!7Gu4=VgN%2IbL<&3Fyu(=0>xr(nkXsp)hq zr}c-5{L0E7bLLNfaPR>B&%mpJcA&5?SYf=6BHmJ#e;}c9WYf`HjZ@HIW7nNE^pmxy=(0Y^!2S8 zL^VJX4EXG}TOQ$Rv`bbvN#keI0+Hl0v}x1viJ9wJ>r=l_M8TW1`<-77pW)K4c?71$ zngo7q zkd*VGS?b~A^k_wLS`;gXO=&YlOnt@Vr(;f3gH6u6zg1t`AFA9ylZ0P-18)l9ZDn!<^BHJuKPL<7Z&ukB8&b#kX!a8 zF4ON}-VSLKu9o7P;2xF}qJ1+%5^D?G!@A$&k{qp3q2J0K5tUNe*>!b~I+~yF^ESne zj+H|D@u&Xnwnvew)lsOJhlH%Df*}9k32Yi;Kye)XAbEW1S?>T;Dumjz*0$bEem#NH zzo~?c(E7ARsu~ji9VX~n3DZq=ZBxyN5xDj&>?*)R7KtGx0no?TrfEM^$|)mx}?Vt_JtdW z!XcE98J$4lsaI+BX#5_u_LD*>o4_X2Mcc_K>54kC+vx|npO)A*&nAB2F-XXoc{c4M zcj)HXv@C(&Xj`$PL@Z$0ckU}W0eu=fBG_8Q%%zR8B{@(#h_jq@XdR77(w$J8p_vJ< zxVsg1>QdTJPx8e_)A!dFH*$WPKqizS!_ik@+;*_q5G92sCJf?kOh zq-)zA5RPnPlm*lgS7mZuZH-~$P*CIxRCkWSwGPwmwHHru3EwPC7C`Ektm}Qbr}fDG zr|r=~K8c~vX8ILVABZKNm}4XKB^#1o?;l00&|gvBFk*KYCc$5yjhzXl(i!6Hwzv)t z(v&)`D3D8NnQI*Tik(0l+r^DnYg37m?Ly8A5cjIzH9$+Gwm4lpXO$c+#Klaa@Irk+N~-RtCWjHDZ{FC!No-T2X-PceC4*`EY1DZ%gwXL z#w{L28L)&b?_h%G2BG_WEBv%(cE#%C4uOg!X!GK~J?p(h#cG;(S*0LL3b$h_KG1GnREMrQ(lXEUcW@3 zw8|ORc7^^FTR?WE)-sme)|!+h-BRxk-RaT6F^;~4`wS&J85esiUIyDShY^b^u2wiF z*E^YX8KuyD|Hd~Dm#rcY3kN!Vs{H<=b%~?Mc>n-!kIpp5sw7XJysm@`9MJ8q0z zctgGO`I2r~7-p^K6ZGwel`2QvC4A52dx@u7bn8bi=QMvM7%WsjGjU`iTlJF&SA>+= z!L?pP^y33Dh4R%K{R#7tMtSa`Vk6NFK|49~_PriEb}L#$ZimO2#;f#Q4vgh1N6dEu zx~G*&ttzzts$Ft>(Q#>;&%N}sOHN<%rOt#fD&}^vzwAKH9{{02m2lMid$JF+qNY>d zq0%}$v%W^2@IOjU=%o7o8}ZTJq~e*YH8JA_6;zD1ekAlfy}Pf|%UKgT=K|M-g1nw) z=y${IOf9H@UNv4g)+Byr+key%1Y%+XbUU#1H} z`rFkyul2)e^)WkNBS0{SRGZW2NtYa%L7QC61M?@?cl9Z@)Pp4GxSMQcJWqZ2On-z( zQw==Yvaz~_Y=w*YF-yH*mL{HMmNaVXl3Q>gki>>8K{f%zFV)91fePdavfkDV=j}GqSbd_K)2T?x+iykF-M;MC`1C#ZcXN97I%mcneL^hH z^-0$L*SB@^_q>1aMtjf0n%>bB^Zn%@vxUVz3Rn|str~OMowjT= zhvybs23$g@ebR-iW^)U_zC;Fs9f$vT2sdvk|yLq|L13Y^sv|u=o-jARUXDTYNZsAURgE>!p+fpX%n8pwo|jaY7WJ<^!-9?Qj9niLGd&JghACQm1&&B66q&y`QK^opBV5EMRApD zYk1Ubb%ArWXM3FTi?;vnYe?xU7?5iH&tl#0%=RCo_w+edj&|XH=DnYa7p$nR9P-(1 z&OzSx7aTvR0)F?AOEI_IkGj87-d^nYZ#BZ-Z_6Bg^du@Dp}7+yA>&w7@X#rV=cR9M=NN00bD zwMjX;k69c2+3xQJh#!TR|2HmsZl7WI25*x$@i@2RD^_d03dleNjQ6<}ceiKLsYL5H zkGfatSIK0VBBUNhRTx>>{*nJ+qo8}dxC?8S3-(B#1zw-AmKNn6X2q<|$be$gU;Dy? z(@BgXz?+rbf1b}R+4uZ^@)xZU2CM7 zP}BxZ^^?X0#SdKYwC<5(>|8^Ou}$*a4A}X5Z55Vt?3I28dMaGN42A*~v!}3s93n9D zvPBAQ1QtoHn3>EfQQfQcQM7WoM`a_Me4X}wicybaxWrK{O77k@G+8@R;a*>7>%(V| zQomFizzWJv>sR znv%sQh2xynNj+RFd}>`^tJ8STsJN1-5kR?s_2@&U0m$@*;D*2APD&$msV+R4cv10u zC0!cmVMdhysoER@6cDWkgvS9|{j_~!ymq_0&m*P;q5U0|SoC%B_RQ$)>cKe=beZ4J zTpvqS;C~u8&qUFxXD+8(_2&F)+W-mn|*J)z-P+HhE#4 zs;wW+U`k(;e0F2Ykc<{gQNiE7oGjAz*INEiukZb57WRMqsqYx6zk<%Kw0F9Sr zP5`EjF1_Z&=SwYsQ2P2Y4K=~|P{^7YLypq-rO%F!IaTln3>=^JgrK|vJoyTtF+Kq%N}3qc06Y!y8ES6UYbfyq;Q1DMK(Z?~+3 zl|qV7LlN?v{D#S^3wlCzt+N~>COw5ggLT2n6YTL~u?SrYsosK?sNECF+XHPt$V`s` z#i8K1nTRc?|KW6~i;@GHx3g%aQOIh~T3!`E*nMcBL!S@aq~Fcn7cpypJV!$oh%4Hq z{)4!}i0mn>6akMrBxo*B<#rVMM;dx{(fHNhC|a07FY`YIJ)ywhfkje`by>1LOnOOw zAd2pV-0kQhAg^ey8z%*yZRObtByFN2{2;23eY^iIT?-g-)oQptr(?ljR>e!0<$T_M z4Cw|YexQslP)$~$X_6*;Ba&cAQgGA=GNX2olk9}WRNxpz#F%syN;6v{|L2iA%})Uy zuuc2t(^&cFW{oS0q-c$2*KYg-FncySgYqplxpy=nNcg`AAUj3Ih~^nOWU~g3)3+n58P{pF$jJ0zi*OfT6a&fA&w;Z6cKa4_r3>`rs9{V7vuqg3{sNE&$(t z#%Gq24OQba=l+M|>nHffRJZ>0_P>y={B-r^my^Q9G&ql6fBShQo2B>f(RI?MEG1l_ z+w*Ia9Z-AOcbRSb;`5(CF2Mg=lKv7dZlV?@p}h4w6x2TQm+|KR4n@q3e5rj-LwO~M zLDD=xl5?N;^A*MSd<66C*LpPa)ZmSlH#32A#Q!JGQ1@=Z>5h}K=4B?97wp7H z-TtT&5AkuVQjifcyKY!l4{eUwnO0yr=dGP*RJT_U(rH&#E^UbuiK6-1f7hj4cyNi{E~8k396Ewrn^|7HgQ0mmrHi^B z3)*Lh6}9)RmzH-FjkgWXD;9JLK{eKtwq!S?T|Q?jcgRBFjwSc8`Bz&s#<8l3HVhsz zys|I0JieztRsKC=34eWwKCI>dc z!lm@$RS0E8_n+g8bC&C$@X8_(hAq_(9aC3@Zh{h4;Y~Z&c1t78cGrY;3VTKvq-CX? zW;IUX#iM?ka`$AEZ>qk2mYn!u|5kDm3%^Bl|C25}qw^e4Jx`Y#*?kpL$cTA7Rr@Mf z)=-zY5hxS$EU|VaWv-ytRYqKCZfWe=H~Dxa1dx9_j>~ddlk|2zUw%YugC{FuR3mg> z4rzIMIS-skKI4_}34Q<`UOm4`hXXGaqiuR=^bBeB4w_c!4ZQSd@1%k%Ii+1h@gP2- zbqc+(+#(kLs`x!D`$==!QL}QY`njvCkxeM$jUl@9GA(Es`qjY;{K|Lf)D0eU42kPF zQv3_GIRCWXEg*kAY5Wl>(g|&a&H_$Pf=>Mv;*T`g%9 zfP^5TXZX%=P3pE?I=g2YN;e5rtLN^Avi|bP>!C~9I$M@2!*!gU);&krMJ_Tyqj3oz zFeCI1h6-x@o<+$sU&$2+TEY&{^f$Fk8dgIXhM@Y*h&2hqBDRo=MsPj!YR2~^sjfk8 z3c>QFq&ZM1{V1$mgT&3QbPAvYg3~OPMmmBH3Rij1n@isuth!?t2_VWXC0puO?NC&} zas9+mqsxZwNKRmjEz=eWI5G}BQ{ff2;l2_vL$5mDc_UDyukSi2U;XBvWetcGMve%`Q{sd@uw=%um0SqS;GPk<1QLR)D8-#42rsD{!7d>R# z398pO~Ui^~im%#npoa&UhkrkCc3-t~NNc)EqzF>ul+R zXNNBIAF-Ms#v2V5*5ctKJ{L|#FFrhSNMuC>$v*KZJc2r$dedCKW5L^r2G)eMAg{`+ z`m+HvhmU2O1|+~>$NUb~-(JE$-*Tl9x)yKLaLsIBLo|boqIGuZB+OvDQh?0MI@bzH z&+TKZp&h@m|Je{#AZ6(ESmsk}ySp|JZWPTW{4hfzek$Q|G@q0U|HW<4oFmobxZ1Hb z&+scVW^M3?07Sd{2g_=@uWpE;-N)%wE+-{Q;2I3*@;w1AsE-@!vnqnJ9#eY)I#^yl z$y^ezX80|OILusYk`=dCIO_JG#~oGMzheTCeLnp>rP_+E_n zSAMbLDqV{(BT(tVz!pAHkm}|MCta{v#x?A!ul=UGIw#vU|Mr(`_^FK(USSy^wjGxn zJ3BKG{J6M$Hboz$WT$GCM6gV7Us$?epB%&o=^35Mj+v`Hz4Q4!WjF!0dRMr?*j&)- zd_cln@frPX3B?52_$EPjG>Q~LydDM_TqSPMifL&K|Kw94=k`spE*Pa6+%R5ssfgK~3B)Sr+ zP}T|^8?7$khXCh5q%Lv^r;ifsJ#Bf#Jb>iqK*;)$oTcR)PG0fbs^4kpdZ2$10W_-~w`eOk8c9vdxJI+c`qOJZ8*|r)k!xn&8 zdNNX1>4pV{<)cXLyYZ8 zeRCAw?=qum|8*t-)XDLf5(r1JauA*@Q%l&wJ3o{YNPNcnicMUf0~ib9B!U;_OR#+F z%M7f*kD7jUR*TX-AgGoFnOB5K4PE`J@09R+HI$3Z6c4}OTPLbfRu2?jKR9wG`bV&k zxbBpgyvv>&vu}K|*`$k@(=Ze@;JQ1{eJhCHz5j~eydeW1+N}f0AuVVUuVbO$aKz<( z=V)YuqL$7C}m=J5Ob`87Ug~ODI~}nbcbqpY4J&V+8=AO{i6OFWQJ5 zhPlHqH%+Q-0{lIpc?vOWW6LcMs<8sG4=gN;v&-h;SAHzbAjS8gHB)le>PIB~p!`)ej`cVnX7@64_Uh>ORUf znjJYb+>N4EXWpil4k4?*FwvAqQPbBwOWBbdSLLen@D+43e1KQ%fpU$}TO zV>pVtus!sser6 zH6!ME=|wL?7FkrNAL-{>G_+4l$e;oEN>^eYEpvTv3%7o?J`Dkr5$`!yd1=Ze&1(D< zAn)C(lJ-Tk;O+qhBb5~S?1bknB!LuQe{SAebgb~2)uMu*bOj@Sq%`#3po@YHV;VVz zh8eCp6}wkai~R91jqa%Gcp-Uhjf+&m*OS1C#R1IAA6B@KLUQo^v42;ZzT{GQf97(; zr*txbHrw}mS^6X+--8|m`e>3FtEYYq&YD;_!6@$;%0u+t;U@#7_M2jcd4|dO0O&q> zvbicww0ho(soKPu(M;=CF8Hw`#F?yCt~_)q}e;-)70c+=L2>+BV=*xt5INq`e@!xI~{me_aE5 zT^`SE(y~-b!hr36<;>0uBu3_0JgWF2w2;1LSs=o6)J$4L4dHtr^JL5kwy<%4zQc@{ zQyKI0&L{OAO9+uxh(X=#yg&8*6mDpd(NEUUO{I<~;;`fbHxSxhDd|@05!r zzgRB4Jb@dtCABo8yey58plbfkoHQlBBTMBvUo-Bry7#*|?g5lh$J_PXzEe*U6zDzu zXZOFlup)(w5&iV?K`V@Rp1tPz*_#J;64>wLWlq$r-Uv+C16q@?E%d!o)f>jPrFsM! ztQw$GEwcr|z0ADu^Xi~8h1AjyQjWffF#Y_T=KE>G9k4?$S-sQN<|?W>t;`xL&34Gg zuzN5l@I3rfMqj3^{hx})-YppRNjl!QJszZKU0r~i{1Gia)6%vu>sBvRMaCXjpMoxW z3^WKP21kJss4iTC)PGJf?}N;pyUeqyTzt&E)Qut`5v)Sb)uU{*KnJvUhLCvMijfu- zHBeTLcwlei`TegBu1EtJ@i~vGASoQ~@b0#4S_!irGCtJTX73#s<}BnQZI^sp^dJV; z_L468UOc3jPpNWkSN!8TZnkR>F_kwi)SGkyiGqFd{(C9%ebF195|1~U-+nf6KFud!)M`lOYo`1gc zpGmAg-v9p$*niDRuR6GGH;*hFW;Tg-S^}>*(Dldhp?ec=>>^V*rj-|#X}nbgRi!bKkVw9 zPFeL7cDen~Wm;Pb$9Km9qNIorXw!)EDAWqJ%s6}uxmHj?JBnOqTLGZ7rRxTz#|Ynf zsQr&lFcNJ2;x7a<{=h*@OFwT7(kK+-`P<4WrFogyWq1@%RO11eq4LIouJsM)$5gnix$s(X*4W7ey70h}JAHQl#+^P8B4|d+2-ZY&I-hq< zWQ=P1!fLc+o>2`lv3<5o8VuoE***hdZ&HF~=X&4xj5isIw8N0~m!y}}!{8*oa_Wyr}zL0FUE$o-8C8g+e< zTliH8v}VQ1NIQVqpMjl+4=-ypKq{XrztCsnP&JhHv|9x|6CL!Zww1e?cn!D;C;x(% z0CUWd^5=gvU-9zCZL(v0Ke1yZ5tm_YL^JA zT851k+1@Acb>islqI$zYnhs+BsaMK7=q3q1Fwa^<1uc&v9|oxsS8NNga+K=p4T(&T z>pT{YN#fe3xL}BLjZI%z4)kw-eX#L{&Vgav{hHl)-p4i!z!T;z&ZCY!sx1V8vD2~Q zjQuykV&m5FAdv&YU)jFdFf?%x8>A?~w=WKGZ2S(byaSGGyoa6P&9jt@#OWpQLCI@vdlZAvB49X zO27}qLbZ%~(Q#^H)#wyIkcd5%XyMvbQ>W+NY%1{CWU=XELQBM1?-^?~`H`SNtmT7a z0pC|Tclds`aj1ehw;nlgOyhZ3kS#(On+(S{Bi#FO!$S)=S%Pm&{XinckGt@2VZfZ>SzZc_e zS|h&qVTJ85+>^ts^tb2t$)De3%Ob+)D@#vFRbmea zpe~06csi=h3;2gJ3YhKzYU6S^}YaCWq{K> zS$ZBdLuphTXGfp+q9JJOanH%v&SmckLkods)GGb$kA0WNd=vv{y2{x3rv6zlKwCf% ze%$fut$QT}T4>wwZ0Ne=VRLwO{-KM3rp45j8iLDUH~W`TQI#Y86`pf;E;*Np#>pCX+>_i(>1h!Z%l3w38TOq znC_;31N0DN`>#`r-+Bg1B3|>H_@9rpxLUF>9FYA(Sg^~bkb3hp2Y$~KYiB2=3&4Lv zqP<#XcZYLkjR5W8dkFfpWcS#h#HnM*-nrFPk27pil`EG=7O$DzE0Fq%f6Lej_Sgyb zfJa1Ac1qlY(!-*@o1Bek?H7}uI&WunUAGG_m0WUkct4JHzEFY$1a%K(4J~!uy~yxQ zwg!r7fQ??muF&xGQ7|VDKvY><4(xfZYFOfgJMx_?d(U*NQqY9Tlru!;d_`=fshFTns^BxH$33A+_+8EG#E~QHE8{2`F_q3i*vicJpW-raq;R$Y-HAJ=fLZa zs20;&B?D?Wq{Jr)A&GRRI{@|0`|_nyo@YWCHFD_sU6Sze@~V}FA7d3;Jy6Qq=? zRRdus(L8v1ZeT(T0kgu`1DVj#iJJDRlwmSJb8}_Fy=%HDGCj*P<~9~_C7WDXiPTe7 z`qEoh4}%>*rArNNBRDC#2|Vu&VR#C!R0WmnYZe^1r^gKbcNA?I0hGo|#1Hv$rp_gM zGb^1NY?*1zrWb-fDc7i(IZ{GnNFY}-9QibT3z!+&W5#SS{5;*_*YDjw*Fs>N-_SL< z#!}@M_;f;1+O71ihBNfR^%U9ITe|{&^#0ubuH^tG?W`IK^FIjVkF;D}s`$0s6Kap4 z3B$jySAG8s;Kn%Pzac4Gmr{-{h!=+#dKU1KKjeJ!sv0$_O?qQ`*{LUQxu({t;u(JG zFMD#LAD``c7T{U@Tjpkrb048c?Rv9|sW0nx#Lq=ehTCU|!V@g)AZ`l%T}KFJidF-V zE>cKkmKJ?UxT|lmFBm0em_$i7k%!)g@6%#3j6cGK$RBMxp64*j2-N33`>|#sqI6kh zP6n5KA85`0N)0l!K18tM1^eyQdezaR^>@%s-&U~ldrzudN&u7uz3F)>KYv?@#2M>^ z;{{`~y)y?+u8<6|fjB^_&7Tg@pfA0A2%o*;uEC$GVjwh4-;FW5JV^d?jnOf+mK8@HKY0^5Q1NLP``RbZB#LO+fC}Ta- zrZ4RTrow_zzJ_~{N)GnsizOS>Rhp@%Q@Vh3c`9gp{w+E zoOxAV&%$xQsSPz+pXRs*fXPjIXWj}&PC`!4J-w>H_scE^5ae|OiyM~Zoe56Oi${^Y zX*4k*>GRS;rjyvaah4!3IhLEMaE8w4+KGYNY#ILEd<3HgX91T{Fz|M-% zD=UXuNVNO%3sSt%N*-r>)}~uXY0hGMaE|~Zg$#kopPxw0GlM2fWZcqO7k*_sM_f%1 z_0YniPuauXGA!qAHqN*So9didqwFO3mQ8%8&$1yGL*fS!dQ!rqzmDl=V+9JNOW5GZr`IqwJam%^9Bn^_wX%4|nb!_hiH>)F30- z4!TMU*0gS{VaI9S)Ythn#YoaIN{gk4x7&KT7WE+WR-k~{&5wCVC4#2np9r@X#5?Y{ z{GR+`th42|v71ECJ>3l#v-FOdpmc#8NKAWV+p?|p%mL>xa1awVTDRs6 zrSdBtYc;N{_rQIcUyKQbNWEQAA%1w}418<(#VbiNo%1}1WyQ!Sb`j)s!M-x&M4#D+ z!V6t`0hgF?iZt^B8xs98z5NaN%;n=C(MsT@Ax~^Epc5ZaNGh8Sg<(qXP|%MtF;770 za$76T3uQcObJtRSk*$Yq^KXjOF8)y&vq#TE&40RW>d4PZc=4lTYTomo`*1QF>R@2o1@| za0c7)NdV}Z0TvtwM{V!S8nd2kA_5g0whZWV*eAt)Xm^7Pz~jYUj3Y>M-K;^3*hF1b zhq|gbk~!ZcNN5jpzKiyPCyPDo(4o-ih4Z~#vu}5fs8@Dk7TIfQF8vw4_v0>t^UA0O z?iI6;!?_roHX{MA0+qxE7)I6r6mS7G}V zl|Z6aF;a|lYLqz4__pT<9mmGVu+H>wn5*|t0eoDaOU+H%VEC(6Ss6?dP4q+ zmaDxHhz{8rNF7ylOOKkfEc{~bvcmQ9b=}uDGlPLLw|72u&7|n<_{X^M^gJ6JDF!A3Dm)_hwJjI9~MCGW5wSzh(5KC-B4?1 z4ED()*D0wWa9!^j)N-BP!2&|-N)1iE<$zK2J5sSVqhlE`>R#>x^zU9I4}=mAg_U|U z2`e>OAp*(o3in^uWvYTT==TjXLvY(#0smaGKOdSBBqF^K|k zY^H6MGRZxv&|a)M=tbknY#h8sRNSUBoubfbzy582-@Nb_+g*1Tkzdrd=?DrkK!aN! z)lH0SU9AmBEHKC)#w0yrwrK1LKG^6Sy?PwkTTn`i;Hg1MJ*AHFBd<_IX>L2dxS_aV z_t#koW=S=rKh}I7c#cA>A2#jQ>T`OrP(cZ3lpx$@KAP{&dyc)#C(7(--t{bf{}%JB z^TDr%7nTwoYJaJ^v}|nU|YR(#Wt1f zj$VpbBk7H^+cwi79xg2v8@E&TN#9w*cRN5B`I(e{g{;dH>!VQT(Uo z2%euGO2~4_3e40`FXormPx=r}--O;$0|Lh4YH?c5lMXMw))Y zgi*xu(ro;NU_v}R{N%`IuJuO<5AQIMmb~DSn$(%KGeXD(vOf`%JxP%$yghpVtxm?? zh1FdI?>POz+XiKjua5hH?;o5(%aW3^o~eGQagTYhXmK{)ye2rZN;W9a)v$lE-mIz% zCVq4rt9WntAKNi+Yyp(ns{n^MirQ>0Ks}ikE@tq3Y=%F1w9spNTDCE$6JNhwc zIm8C)Y}*=DG5jSDV_7}`ue%M84Y6Slpk8d8rE;m`^LmN?4KNzE=4G1y!F##IlNO)X zvo5xQ=YQVPTmp}F(h9}|&EE;T8qW@=RR$FY;L78V^9I>qK=<3%XPTqaZfllKDgkM| z_z@y?O?04U7WBK-XpLjfL7vcMdR!Skox_?RV!UF)N^#Tv5s=0&i5TVOZ}kWX3VX4V z0-YZe_)jMg`x=(H)l0h|yI$<-vYy|Pr`m*i@VE_51d+Cu`7w2U23dD{kVv|gY28jC zx>Uja&g#ZGdrpHJU6_*SF*Q0W^C7NOBIn*=hE$O93-m$ZJ;mna@j}27-nuN@TnRL!^I>VX+|bE^#sKz$ zHJvv-pCxv=p;q86&%V}}R^ImgFuE*C?zB?e95R}`%-#6tB1;diRYg$L|Ha;WM>Uzf zd*9Aja4aArpi&hJO+W=iX_0OLQBjZ@73n3Q^mfmP*bopAP!NKmfV4?cH%{swh_HcTlN_nBkL^xATV-!6BTr*Ky$WmT<*sPC& z@t8$!w$payNT!`-i5#8fe$T<6T2cnuNYYNBp;WF#b!qFG)>Y77yRH@}zIzt^D|)K}g8ut;t7iH5J(oTol!<01N9Z5e*#!*t5_CJJP2$wIzM zXW^+(_IJ;5|GAfY(_7IFGu_A}oPjyvK+OzPb8@yCw`q;Y!dD2AV_&~N$o3BA_G;G( z+3TejA0T@N`^j;x2kC35v9HI2*ZRN%dF&R#PnZ4kf%rKN6qfd-RAKmHEMfuP{e}o` zv_a+}q8jvyNk)I+%eq6RD~+*um&d9YdnZ>d|If0eu^=5bo)qF z^bO(&KR{5TXk1j}i+C2=wDnoA{Jd!tR zRXHaJZ?*G497XJx@g@dsJ;8tK9>FhR2?}7wuU|z0yT|9wA@2A+N{n^S;MsmGaCwyQ z=07nPa+lh_Z(?81YG6m~9Bf4(R!a|VJ9&W52B-a+o}rjUlfpvnB^;s0JDiJDHL1Bd z>zJhZ#?z-f!&Smr)%WbT*NS=FUC=HI$}x{v`!R5dgotEjzr4jLh=bi|j84r(xA{sJ0N%#O%k zxEagLj_Sbb_lThWmgTzB%L(x-zcxo>DaG_-$Wo54LEq#Mr~_FGqAy6 z76%GaPA?7mSmvINqEqy|>9)qQY1ZCM3hPvylgOHF?d2vdqi0W~Ke@XTd^;*Tf)bzA z#MKL;oGDa5%{%7KZkl!1^pshCtxo4%8dZ+nL@U*Y(A(rc2yTYiOkj?9jYrVyvvTuX z{D4ImTg!LBT7;L^;+ZQMOEvi=FzC4tncn z&|5^%Tapya(<&e7apkg$8U2d|JR0YBiQ#9QpT2f9K12}hl5u$(#wXAAxY;OG$apid ze|;mpK6-xAThqZC9SsvmxciE=P7}_IcIc0oOn=KSdho)iKOpadf0m^7nZ=Z_$~N@0 z51yJqlD4JxAuD72CZ&Y*}TBMlA7mWc{{5<3)5^+$H^FqQM&ihcO2C% z^AkMnK2h(ScaPeUHL;+g%+53yXp@_tMXJ2V+&9x2#BK%OoU9{|l5_WFPkYu5Iq#mGMUL>2BvbI3N5TdIXg~Q9tC?1M(73}os>G4=J&M?&6w#Vb6TKOxxVyGS+2DC z0}~Fr&w}3;JV7gGTFFC;s^BM(jR4vb*2Og3se_rApmSN}@f$=RUb{u96Uujzhq}wv zAgjIhNar)c6?K!whxlx8GO6h?+htT$7xss9NR%9LgJ|$*Ax>t$uujRXaY)KHWk!K0 zGfuMGT9$w9r&ob%z;!<~MF z+i%iZiB8#e?{ZK-tO`tw91=}>Sc!B~TZAlgm%#k@CI3RF)T1&5@rmWl+_0>?9-TL+ z={S>VhS={S1@|1x!6JH!@3(9L4%CwsGk=y!pkL&g&Nt3SzZo&Ld(81#YU%@Xu|FLb zk|=tK&62;k1pf}^2$OrE0ZvXvCsNto zm@S6$h`FG?F?BMsOw@;lJ`)WDo95#kX@(Rl0~EZLft94CQqA2>t{82=a^W&UKw>DT zKW5RW#nn|uWwmznVR3n}xtWEsjStJH-=@gd{huske~3QEXASF}7c(7qukLw)WY0Qh zFQ}aplfgP%UQ`2TvqhOo)KwKzTTH=tTzk>JQKz9HrqfQSn=`;BpWn#V#Tb8!f15|W zQfYN;*Opp0OTJfI#m4T{Z^}hvUyR-BCPlaeMK3O|VnDJ^;Sk=;zS16??nk<%J>i6J zGI@3#EN=sGAed(S~Br=%R9W$-SXQN^4H4yK+D*;rWJ4Agy-qiDcE@~TC}waH6J>76anq8x&S9Cw*R@D&F{n>@oq{c5~pDD2hE&tLbJAAt~fgN?Z zegaB!A&^Jk_2>G4XT}9wNXOw9vcGoSbst9c--v0RfSMhw)}Raixq!LDGy!3L{7JN zXkV=lV4+&!NtlHLrMBjHl6fM6l519u-K|7X@V!WkkZC&@ti^#yiKGGVFWUV{I8cJ` z;lg+NhhV>Kj|bew1nyuf%l>KNOYNS+#R0Qm(FJCnnmKmN;*^+8mL4W=vX0Q|n&g)A z>{b(@Mb2~1&1JbP&nQlyU1z^lY4DMEL+DV~r)U`JhPuZOcFbSJcN3qNz0bzILC?MD zVamV*Z-`A^JT661&stE=WVs_M9c5qTO?F)4OTgbh8K+awT!9_JCUym|_;c{!e@gxuzA+JX8LAlv%!Cm$8y^v|DvLzZyNU0n* zAD){hms9Z81tMlbae3aP5O6$$H?J`vRo->cw-+;B%!YL=QZOKT-G!JDyEQysZnutD zF1fhTS6P7gGAoDK`_Q+;(?HM48REY%`WCytKKlD?*arTQxeQsz8z&%);xbL&GPKAu zV${BLVGPdubtqQ2E7L2-uk13E1`rB2q1x|iaT#7M1@)yDu^*}P4>5JW ze~~{M85v+@y(wF^1Il(?t=#7EB{Wv%`hJo=O z0FQx)qBbO9G*56>4m`DXvgjv)%)@&%(s7fK8x|=-hJ0!F3w-$8Stjx(oYn069s%7L zja?s`(9wPuIn4<5%Dg61PcUX!eC;RYFr z@sQraoqn|f3({A;AbYEvW}ASAN_dUEb2f1SH2PqZxy(Wpc=yG0T!&O9FDW-J<*o0F zDCy}fG;v>|{hD5E5VD+9_c?Gknej<#UAn6FfLTQIoZ+0o$KZxDXQ(*gG}^Yq&<*Yz zV2sj2@Sh*@tyjQk_g#@vIiUf{0gf7ciacKhHF(a~ug8JaUq5wX<*9)xiO5m-sUa^# z@Ux;w4zcUsFw>M~qKa{dpdLXhI-;lxS@8N$@QyFi(}R@929JZ{K=^%ZmywFd$kyVL zp&{17*}42=pnfbn&KNF?TC#tSN|gr)?b9^}lZC0#+ak5=n$J$3jEQ1RT8UxP)m}|9 zeL*C2Fq)r+obQkKC-5i{s$3#2;s7bi-5yD&NINh>>hAY5@W%+>8#8k9e*cQ7W^KEN zt$ezjeLbO=?M+$KFlFGXX+lyPF8BQ6z#!J}l1=!R(!ooST(BLj24N=w6_K}P7fI;| zt$NsEQ#=?%7c+2IkZ!qsl(upp4T=W^C1QoDGZL^(b=!(=dt`c2Lc8Ve^JAtTWGJ(!7DTtGiSi+DyVt z^&|sigW^lVCtL0+fI8+$L!iL!TXl6q4mk5ARI5BvHwrE=xqM+(w9}) zT3{jgk3Y-*nfu5n3@pzYZl*8q{`sEp z+_~4tSsYE}F-pvxCTsX?)GL&SLR@UU=cpUL>xNS7$I!xs$mesfG0x-mTGEyk>s?+} zo^G<3b1V5nok5biuW9EO)QKyNIt3P`A4qC}rJQ+?6;|#jg#UPER1%kSq?$)KmQ%=O zX>PaFM4F$(?pO`dLw1?ussVD9@a<)gsdu)Vm;C7?4F$jg9+8A|0QqnS;QD@AnCI3%orL#~`*Z{sN#H0Hy`^_S@7v_BtPS>+`h8n2<&>xGIK(L#SyT z;B%SG>@S^YM(lb>d@C2&iIYcBE*zVyNfS-B0>9nGBibcyZDu(nTK1NoYLdJ(N&*HY zyDOr4Gt?676C?1|G^>iJAZ=10=Wr6DhlgBU~6T{!{lT)p!6A zI3m|d&a9=%frhwU=MQX0JQMtq?9)viU-f9#O%mHVe4LbV@+PQJYPNwO4BZ;@0bFJg zsLqbXrCOrKpK_KaOR4h(98x4%1KY)BVVY=#@<1M zrRqa~&H^I#I9`r=cx z!c;YMjzNZh>vjN|d<6Y|=a>hDtc!DEKJK?ol@%*i7(X`W|EdIO1~!YQfH=+eE_dyb z3_z1ty7oH%wE1ha?S>8OHcU*H$;4IJtM8)=jn)~h5r2|!J4P5|5Rl*YMn(ojbGzV? z38#RbE$zf=e+rG%e<|n?nqwQ*9XP)@Q=aaK7^C6Esw4fhbodfgC}px0s$lp(^U?9C z_o)Di#{%r=m=U$K$M(r{yZ*NC#I_jBMJHv^h{h=6Odm7XyonNp4{w>ifGsOMn%Ujl zGV}2FD~MNQQ9&g`o9x}k8VhqNZu*KRgHpBlxeB^W(}j9~KR#Nx`0c3M0^8$z{C=Hu1ZAV>BdYkVB;iScLO(T7#!~iOUdk4}U?=Z0 z-5=nj2)cQKs4yDBhxJ zPv*5xIt*f0gP3kXcLW#Ap8!CUU`Yc&vpeQ7LX1-;A6k(kza4=e;u?G}<52;41Z2}Q zkLLnRrXa+=#lckg^4R5R7Rt>VavpFHf!$f$!KtsHCXWIZ08)LOcdQmGFq`*e(9-~W z<*~E|t20v^Ig_4kcZbdwL%iK^D(<~3 zqB3#Km)TGd{ZdKIgL?oG3y;UIS$+2PZ^i5=pxPjf^IOq{K3!H0RJqlS2imf@S|mu+ z)=l@bKDu;IRwlV!_*0lP45He|U0FJB1i&dqE7O8Rq7FnG3AVydz@jeTwy!n7rrX-P zYn`9--@(~6|D0kF=mZ2T=nv(aX`PX9#;^Jswt-2b4g$7N*MctT@FLw7w+4}LV=pT{ z%dzdA%Td$&M6?(BLLUOoa#~=CV|J#IFP|}Q9JjI^-i+61n%631+JHH{7C@GP^jFba57KpwMF*q)`!-33B=&|MSP>48u4 zb=J+tTC9Zm1qrjTIg3=(NiH|$UgF;5D_E56gHq(*)b&p|e6~DF0!tCfNqadNsy(1& zl5Z1${@E!{thLnWrhJx@q$i#+X2}fUv{z|&IP8~GMJ0J?=m;#WlAZR6gKo0CprKRm zwEOkBAUXoeULbHNRR?I z#ix_pB`Xz=2F&*aIIEQTd^pm6;cDsO^0{6{2UgBQRMtQ-`&8u|9xt3mjtKq+XQ^v? zPVO>tw%gXq7+cu4ko)L5sBVP`e1zXtt~cnVGe5QvsmkmhB+tuQBt7&~fv(Tq0hc~$ zKJtiC`uw9mttjkz;WctHUV8<92I-Ce3|h@JdtVr4x&#c$`aGza}k&~Jt!!y6|WDA0+c zO-r^LzIXrGfgoTF))x6ZU7|i>zw51EiSgy|8!z|7V--HT8YbFBcQ zbP=~@xPhu`$bZTTGv{(BD2dG6>Zk(`IJ3vjXYD)r5Tqvz1H`VR$J1z`&y zo47ktqvn2iM=^S0qUV#vx!k7U;@*89U9UaVPIU`)=bQ$UR_jUnOd$n@#RWc&p+<$g ziPthB49;Jh+14dXDNqoSfY%fHeL0D^II&Xxrc6gOK zrFj1o((qmzJP=j(bP@31{t^RF!e;dSq7b$L7RoT3Po@YrLtlL!R@(5H|zR$bI%ad z?sDEH^o+|@d@%T4!bdjX^M~g5=fW^kq%}A_(_M5&C%^jy*rWi|oGD?5J@PMC#s z6j%n^y=82iUVzsl@K4L;@Si}7upuj(K4=k5(|`tXBL%%TVi9S#jyHAq9Bc68a&_xQ zR#x3H!$>UJ884;H`eWN{`^b)-Y*R@M(hvyd#R9l@8dE6|MMAGPT-lZtI2 zK=`5k`->!2)6FEK(W>l*#hR4{sj?>vRyyM$4l{#0*yoIrVq>s#)NU0^C@ z{W-cK=(dUglr~?-3s*W0HRMeeD$jdeZo0Hef{5JOH&wI8ZBeX=!IYjXGADV zZ!J`h?U`S6;PBA00_y%>voLJ8Jdf(Wz)+qOYVeMYy7E5t{j_U9fMX2!Us?kFcwvaT zecRL_>RI*ALrzi;yS_baoXfe^noFvV2UNXSs?i2av)iVwO3!;3%H(j|Xrh);`Ng^@ z`g2RwM}B!z=tTFL@EQrv!d5xSGG&}4i452s#SYvCfE(p$l?liRzh8Rna)EFE8Je5X z#4DpbBXVZrF72fY)zMk>kWU_PjMH*48JF;WIUMz-E4(_=sm`X`ws7`Kn5(L2@amV2 znmR@k(Wj0CHurFkf@?Fr)f7EltTaf>t5PQZq@%BiG+5t^9EyHGo?j$^rpw^=yxh>D z#Uy3aYbXB%D1+R&*v7Dd7g;wJoPm^t4ba_1uYJVmK6OmQ52#XSxI=;MT^j{RRYcj? zlwP9tc*Y*S<-Uh?t180y5+KT@dD$CsiN`*DTtV*CsdQ>Kje+fE>*DUtD~n`HNMxvFqcbA^7(ebt9X2+k{ka8OOV;_y#A99I>e< zD>WsAjWa_ST zN&~ylFWn3GMCH5C_&HhwmV2orXr6WIyxrpX-fT3WYyVe{`Ij8>PvPT=_VhoX;s2Fm z{#}&#e?k5FzgLc_aWAyMIn$dMn#Ud2cKvHsU3$@Wt#EXDv%>()_w~aHMxzV6u9dWf zhW6$r=F}>npD?PAR1X5ecp3L_JZoJJsC#g{t%Fc=MiNz-Wu=K7z+#3yo||r0;kj)B z24z>z=r~wJ$r7m+G6mepGy25@6$ll}U6;j~n1a?Tv2m}=QVu?KhL#fm@hSSG*C}N3 zuXIWlvu-NKxHZGwu_~09iEBQz!mR?URx5Tg?Lt@PeF3L z@bOpTM$vR>NQzV~53hhrAyS^=xB$cBhb0H2!DSbJWZL+dfK?EvFnu{&eUbWIKDAL9 zfD9dq&7n}DiWBln8P#X>B1*{}e1IAgqs}%%xHvHBZN)-U z%~3@l!m z;ajO$sh^M#nb`xbLtOPm31z}br(Tf}zTJWzvx|WH!cZWZF4`f-5 z+QGhVQMr47=&#q~b0{=tw-=qODu!R8FYUH;J5}72`8=vedlo)#Dr>?TsUO7?gCL)_gpjY5xZv`W4W%{oSaf$;F}ueQ4xNGGo8%D3wut z(m{70cp7mGLIuye9?o;S_?)}C@9JM#fNt{SdM4wf;oz~xNGIm*oZ`N?yWGYavt4f$ za5yj|4l*e;8;Q*Kfy(71zSEuWlZ#eh3v~e!$f}G-QSTbhPk+nS>_~YjWIBDKUrjiz zfFnGm%l4IF$6QEvMtp<#KZ6T0MMffw<6DTVk_E{!Kb`#H-)Bg4Dg3SChMh-O$a4TFM3 zf=xijNkbO!7+HB7 zfh88v+Q4-=wgV-&jiLt90#B)B*nO%(2aywcEALB={vn*U2kcXwLJc=!$Uj_L5~Hu8AzDc0B9vbYRPQ2gbbH-B{d)qiXR- zKyWP69Lr0rp`zATZn^ zHHk+z?6Fg_*&Ykc2zcD81a9LsOHSfE0yc9kgGtR}$eC|i`g|Z+2BqI^#GT0+J`gf4F@1^s7UUeKqCISAhy}HG~;6&7Sa!6Vr2_Tw+HUO=7g*@NS?T`Hg!N6Ky^( zgeTVx%C$?)fg*_ozVkqE6k{A&4|E53t_Hhn_gc#DC3eJ9FEYVFo2S7}viA^3`U#i- zQclmbA#(?7c-UPloN{nx@-g|s;{t%A9-@8RoK;jLdoC>mTLO`jj=g!N^=zEb)j;AqVfwkJ8u zBw&A@2Ytz!THd}y1V}{0iF`-mF9jwqlhYpat2#JMwspT{bRLN6DhS#5_i6H*dKRww z_VzUEIs|3_T38!lsTQ?e_T5(&Plc^YJ}l!gSIh%vM)0?!^lB)cgb#s(`DEP5q^Ut4 z^yc6u7#i$@E;OH2AtzH7H1%Z|oY?iC0%jmAhf3vMU}%LcJ;YhYs4R%9U*dAeVCK)|!RCxN#p;sU& zw0NqrfW;>eJ*xT)qOQr>Y`E*Z`J=PeOmF+}is%OuY(BpYPZqH9|KyX=F1`kj(_Rml zi4adrn=f6uJth+W8I3$~5+KDW;y7!m1ORm=CC`&Xbmp*u&>B{&8G?3 z-@IMD;UXw%2qQ}Qqw1^m)bb#3!{2bs92%yh0Pxjn$VW@rccx2ey&}*WKlwj=WmDx ztf)R@X8)eG?e$cVdJ~~O+H^0D@l7zCH*yAhVZzqzI*^j|Al9?5I}Rp%1L!=(a{~&| z^y=_!E1gew`_g{K9jilG6k75>Jji4>Bt6oIjC#51+kMa@-CRvyL9GV@aco)R_a}mE zsfTr*))CxTa5lSoim&$Zj0--Q!nRi!KT4|KGQ6;8r2s@pfc%tUzvOI|z;D=9!bur5 zk5SxAhj*opvv)4Z!8+WtcOFpvWM31V$i0~xNuyVZgHZ%_u4r-iI>|vGKTzg>n6V<0V5)h6**hq z0yU&Sqr2cnJ5i;E(K--X@A!^xFmrr$|VDnxR-vJ$1MPHQg$=-+b4NC>TxG1B=8Fi7p-t30WPj)T#2f;+mPgpG0=~{!e%Y2?#Ausfy<5GmKytoHvRz!g{cSdB5um%n1bjM;A$7T&g8B!$Ol(G)pez$JXdS^dh{1+R-`y|>N+;9* z07j@hm+dHAS_1*wh9-f;321&7li{+`{8|Xl(6m=zxtN+hUkJPVEWO<8;Fe9vK5d|) zN{^>mjF5;?2BI}mpHBcvQ9Y0mx|QO-o$p98E~tNSPw#?sx78{0M|efQjiR+(G@&2b zU$bSGoo!#r{McsOlR7$m=CC?dDRDNvKS|Jjx?&Ek)7FL&HCKB|?h{a6JXe)1$J zD00y`Oz|T!-`;lK!%I_%aJjC}YwP5pqis^BH0sNv;E|VZvDt&mZAhBUO^ZtOIz8ts zO&squ%UK8^S<-Lc^1-8IQ<|$AZTpO0!AxEsaRFTc`T7aq3GG0)9mEkc^dfhke^>_A zO*gI=D9 zs4+EjG7`l*tJm#RA|;gY^@Y;xL^e7(Prw!J5)!Qei!3uyWT9ll=S;K%trDW51R%zO zmodBDB&k@dmJ0oU#m7L+d_DX( z&+Ypw^e0ko+|9~O6CSIkb7rk}L9YQ%Wr7<4&2m7x%#bnFghByu>z7&fYRj-6b3NkI zg74ISl4+jz4nO7s+`#ioqlsfVC%SN(RuB%ELaD%58+L*~JDHwJ9&ZnKKH*oc`i zk>J0UYWhwCvK5B>9qKs}+*|;ciad?3o8%qxZRs@&&2BC&QUrwao4qsk*~2QYnP1SPei{@Hv2LB(e1s0^)ih% zjsEZvv2EA=$`hNiSa9+ygFF?J*=S^T8iL0egK;LFi6KzQfD*Y|@i#P=W4oIRVRGqQ zp63~#rwFG3!5qB$>f=0{QX_+mZc)_B;zuxt;#}2RAf1(c$m><-lHNr}b-B3|E;bH? zbs!LGVH@4#zP8?eD-G|q!x8WGRlVXBSKwa<)k`>6sph=`?hYP1c|*gtvhPccbrhw= z(f7rZbJa`^LZ45v+oOkMbe-<;E1vwmROcr}Pb-M2-6s@G9tTpdbz*l%M&k;e%J&9; z#9eKL?M%r0b#k z0MUHpn~KvJa({Ki%kGJ1VRwxh94+Q@8~8F%(LwV2MJsTUVEA+(0VC~FF1HBJsgAoS z=tOP%bFGzDHvyZTjbxTRy2X0*(e&{w*>rWeFOPWRR3SD9;2X$-{<%}<3j7^ia1Ct{ z8asf9t3K6lEFvqs=~(%=8OZ~4#6WS#e4u38h5|OQ=!w~k58&G~dK&|Ccn2?}ON$ln zOELU1p3DW%9zKoRJ#QNJXTQrR`*jdW!|;*Ui0Zx!u4>@LN8|^M3(ANkhb$ z66;LmP2B_urq~y5UmgA+HTsnjEICRi*NxU#?3(iC@oZZJ=75ZPMgzBH7YPrBJFN92 zGEFE+qMC29*#XZ||@vB+QXBCe^DrzIUDZ{6W zmf~aURyv&F56dh+r=xSJ$WdnbMgT~wCTQ|x?_r$5Y@-irzD`oW2f8N8MKsQxY*W5v z7*FOA7p*dYND?68fDiS8smBqLn2|*1uW%&nu*@^SH3lBThLHgm9Ys7Mbsl{CL8$q1 z_lF;ZnqMpBt?QjU4P-jx<}qoa~jA46OlrfhWBFVEeV#hg}6Ow8P};v48N zdo8W0fX4E(>l0@d#)YmI=MYWxz-sUUkkdpg75{R_064A!jakAVyCcO!N;TJIvykgS z6AXa5#TDKMLQeoB4Q?@-p@f1T;GN`un@>CdL1^`nTmQCR<}w!i0hoUHpB?Hyy!&6++kZo!Q^cQp>6!aw{@JSk=I%r|TMwN5 zrbV9Mc(A!bPa-#Qo&UyvTZ40%-v6Qgf9B)PHnRKGyV}mnV29@ipi~&o(~e)52~-rm zzR)m@gLsqP(H@+!CGH}rA6tx0Qntry$}&56tf;k&_>q~x-C+6Mj1f?UHrge51J8}e zPOG3Vs7kwTtsNqmk`tqf81~*LBcNgfass2h-Lve=)WvBP3)C{XM7QEOz&swaj+Y3| zt&AKnclLLYPkH7C#FNiH;95%3SPiY_F%7|HGewU7l4z!R!Xa3nW0ASM978X9$b2Hz!wF}6k0xK z;3$WDW1Ku^+5|W%B6pdi-k;=Zp1G9PwN5g}lb|{dB4&`jlwKo@fX;-peU(w#GJvgA|Q%=Yk zaQxN*0@6D8v7tc0%>VcD&ZNlyWPcx@dV7v{<_>tY&#uZT8MvkSh;dHv5{{MCZd$W& zgCi8*=5{`sn3J@Ju$*>&?;ovW<4cYa@!OCNDxM0rp&j_8-0=vw+6SsAXgim!dyl(t zG|L9-Qgc%0$XC*8dC0HyDAc>dbe+APq*CozPcjjW*lyvq$N@o}w99q=9+;^{F5K9L zN)Q)$?>Eqjf>Cn1vtV>bU?Msq&4 z2ISC3IJf11`TYd&!tqohaJjT4~W zJpWpIX=gqFR~dUH_{wAXLlwY1Zr)2{RZEMqqmgOMi4yzXW z@y0hf`kZVTR4ora@B6@JcS{ZZWwU#V(M{u{58fX;fc*nh0f=~DH;c%>U;&E9Sd=M1 zPl73PPlivnkw_uQWKijXQ53d?5zn=4@j$dM=D~2e>?KcSq@DE9N@eVH2Q6~|Gld66 z_T^9U6tt3;ou&U1aJ3;B6kXgO=qOx*Ma{oZI4j+ulwQw2&nYV7?*NTEPE9yAZkV&p zB%GOM7n;@o5@T0cjS64ZPDZIBWK_sD5r?S0EhGqpR`_P; z%ou{~eQifm?iPu`@|V2AA+lwb0VUs{sm34=Ojj7*IDjGoEFh^LHq(t=->aN%y>t4} z$Wa?$xh?P5SqR+Y`#5z`uB41{)zrk)>f;qVG6lJLZEDFIHi&-B9gz9`GtYFymrDC!U}_PU{9Vu*S%%FXB7|@AVA8}-HERzVjWqW+#=3`$Pa9D zcQo)esn3>$>CpOcHH zNNisw2n43V@Fj%tw-&3bhprun+8qP)x(euIU)sTkjU#MHMlYu}K+9x>-)E_UU754& z83tuqd_ow%HOwxxmx#;HytjpPFG_<5r9*IdK)K&a?ZT-n${;@dfoY1=*Y;xvz?eNx zP2gS*CK%#M_`0}XoEr8$)#c(|FV72X3R)QYT{y1srDddGsBV(E8SaA=07MB68P(e5 zf2u~w;Tv`X_6tZUce@w!^@F@+-}@^_4d`tFN<=RFWQActJ;%WHu$>!th#bQg7tYAy z8I4214AP8w5`qZ2HJ8NQ28&ibT98i?OvXg|Dj-|@c96=DllH>mvKGR`+^D{=lKO{n zydoC;!Y^J(VH{_as*@2>W-h8^M*>{3FQY|}zhze(C5^|KHeq)d-E`#G_0Mg$NdrX| ze@a-Y->tsa8`o*^U6B6U!p2{)b$>N@Hu_jmf0)zYoX~Xm;X&-5*wZv`k|+D66E{6s^EH0A}@;h-u`BU48g)K_1hL-O}s!lY0X2=6?X! zy)lVH1%3!z*GioMkt?;G5LUhAGElLyW`6=Ac2G!okJ4rp`~mKps@vzoc8P}-9C?aP zO;ALi{bJAxveDD<8VTM+XJO!(J!*9Gs*@cNEE$eUlix1N9R%bjSFaYC{_Mf|tL8Gv zgo=~uscwqGsE|n}!1!=;Qd!Kd2i6_FON}G)1X>F!M7wv~L{vPEg0Z@%-6tcbx zj{mTm7}f`>uhng28Ys{tKZIB=TI%UEiNdG|D{W|}DNmHt=w+w-( zuLec)#Vj-9e#?qN(#{(D{JmUfwpK``#Wc~#LZJE*yt?VqZNG&&W8WbW`w#o&Y%y#H zZh4U_ZhChNDKo9;7m<*Tt^Pu8gg$3t<`Bb~HevXUF$z+Qxj@9$E?GCE7^zcoPR6=c zQe;|jc{#IV?97efZu89lTK|da{|seebS-KXkYsF%?$;Dae z@i)(<@9Ro{Gn6j|Yjp>3*AmC@fRKIh_`x#&)%4Zr{_}VIwIgoIP1X90Vqd!ivV~MD zMCuC#wlt0K1NvV@z&Y$D^V975v7{T<)oy%aA3GDOwhmBOC$HxDl>!0w(X|h5rB&Ol zfYz!gYtU1{V>AMvV#3iD*){l-EGbjKNO1zgZ>E(TC-hXEU()@l9;7t^Ae zk)RVMhUYn~`BklGUpS}oxA63Nz|6f1@I82$1|bBF>p+5vfGp(^@C#T||Jc2t`}1AE zR@H?z>d)qEyO*c0uU2%ZVZ}_JhGy=aW@x%Zbs=gd{c5A&EC^^;MC|(U`QBbW%6z3g z0*$E!a^wTM?tIorgp$c`hE-!UufLFN=tchz7gkzCF|>$d%@Cba-UlzF_T)MfiEJF>+OX~Phg{nb@%Z#=_0#tD2zMxtq zqB5gCYZ5BhGl*zj`~Z zyt3Vb@%fTV`O7c=?rWG8=pHMMt+8t#1?)m}(M(@~8kiUn131Z6v^WTSTSatZFNd;y zQX3bF*71!p+65xjoEGn4SJJg~9XtrWaY+5-Sfa>~zEz_f#tNk9`=SMTEs$>9O^P?R zUumFNOU4-@I&8Z!Wh|a7VlxDDT54F~?gs$Ox&bijRF&M7&n+<@e+Ld)K;tlE%eda) zE+dih@($2_ABw06(!wBY05y4A)PT`saCqAbut0kH z>s5mN3!l^UT=$JxoeNOp+~-d<`m6XouUI3PD6w4b%B zdY4R;@-B@}dGUn4zc)b6%V5fh51}OW!|rCc(O|()VkJBI1u*PCEzL5ED<0Wno#lRV zq*!9~5Y$v>6q`yXfh<&3E>X}okO=JldjzlEo(zu3aB1L36K2yS_W^nf3SA4^Z@Mjs zw;7m`Zw@F5#aY)pZO81_)gRDPIs)=SfPhE9S3QsszyCmMy)z^IkDQi}Ki)3ibsNK+ z^xlE%Uz-c|_8Qe2;R|z3LY5H$pK1?X3S6<&m*D4DEcIDvdK?ghr&GnAR&&NIXTfuc z0&6c9LSQw@P68NbkF*0cT|dfRufj{w0A{Lf=F zG$;?SQDh{pX(asE9O3s!TR@&)9TzKhmlRRi_8h27u{{R7oe#j0DoPDpbErAv&Cj?_ zB12?NkIoMthLS$T^%*{4E+!jzF}fja8qg+v?gjh%&~uvTaSWe=gy$$ZCff zC+^QK^?1sCX~^r1&uEP59O8|nb*EYIy(qPippA?+?221|=Af3osNvpLxv?Zf>u%_Q zIe%v&*HD%nqemAC^GZjUuE~Fxo^>bt*D(?Lzj%{Y7h3-3^Cv8=OwRlQ{=6Q0`~KxwliyDEw**(->-CapHsD;$j=r~`Y*To_C%E7I=Y@ZK$nvu&`*d9Z zH~uQ_ex2KH$@t*=L@k}IhI_>hQo*tA4+i-9F_&VrDOG}nOOf>tXSGfEQFaKS_|M_*$&FSg00{FRonm1X~ z`_)MLU-Yo1)}X-J6~@^r{u)-SMvxCC`}P$m~TkuAg}Mq-HL{fmPj- z!7Vo2e~RB<`)%>mW85ddfNfnlACGC@O1A5C7YPWtmUffru`pXIR!&N0S%bo9?f>>8 zY&r5qVsVd8Q%VP-bth+uqaJA$qPWcY(}zD2$5B(ra4zmU2O_IYXt%`vp3y{-^|6vI z{qCIPQhmp8S|Q~8|J3>;XNSzV`(_jQ&v9E>8P^gQw9;e+X%eDYMl9}PY;z9{J2G}~ zBT9eZid^AY&YSszkKes6zO(9S)4%=3;GsM(ieKb~2hJPB5ACUaNn13@ATB zE6kG+?bhd0jRQrHWPRR!W~pp-;t+;)v3|*hvk(=SHT%?mTig|yQ59ABOg)kDT3!SUH*OJ`ga32{_-@I>M zgB-^5J+Htux#(P$Nf7q`~2+b_Me}9eg+r*H~rlI?}~N5-#-2G<=?xPA3xn~_x)z}WO|T` zMPa32;_oXOzqj2ojH>-0TlQJv+wO~R>*Dv;_1IfQU-}qtQvdAZ{xBDfX@{4uzh8Uf z$h)&jhxEjnpSvfN@X6mUIO^Kcq4ccz+r{+aUH`rv`yV+i6$6DXyefNq=Iwpq{(=vW z05zNirl0y1R^K=NbUprS))l2gGs`|t?@znoeq>+7@v6^n88v^K%bnd2-#ZU@i$~AX zhmCQif{IFuI-aJ0%9^W_uipP(b@Svq{d&XvdwVKBKRXKy6fBII$j|H7rmdDXU!1un z`-h7NFkYtT9(kW^YxmFl+q;zC-2LCCD(_gNbg0g<;`c^w!9bgDeRk)+U5@4dwsA-8 zzwhS?q(1-rQ+Zac@U~UO`)9s3XPz0(U%%*w>wSJ`t@oLXAkcdyL0T(3uf&{8|x#KGLFA^`I8G?V}&jOYWytyarX@Q+#d7C zd4*}b3uhc(aJ|#-`R`ZXlD1f_XPXe^V)6S&d|jG(yKduKm8y2trfUk*{PTfjAp``t zZq-+M6SjGV-*W$KqkWe9Ry&r9Om6-^-%jt*)!H|Y41c-LesE0cY zd;Y3Tm#fdb|3AI|{}(->-aH=l%`?u+Jz7`%C+_XVS3Sp5)+FDtn~~q2^!Ie$Tc|w; z_)UTKB-`fA|0(`;`^Q`NewAIay=Nl7{(SwCgEysr0!^&E|NnWPy;2gW+}@Y>`P&|A zNq9B9B)~-@Rk=?1yLEnf#z-SJoL8GrMO@~HEl*XG^fQQvr@Zjs{xSV&A*^bBSI!vY_8 nAmeTpKz+vGwHRgu!<1+L@4m8Ebm%P30m*v0`njxgN@xNA$36ZX literal 0 HcmV?d00001 diff --git a/tests/milvus_ann_acc/ci/function/file_transfer.groovy b/tests/milvus_benchmark/ci/function/file_transfer.groovy similarity index 100% rename from tests/milvus_ann_acc/ci/function/file_transfer.groovy rename to tests/milvus_benchmark/ci/function/file_transfer.groovy diff --git a/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy b/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy new file mode 100644 index 0000000000..622c4d3ff6 --- /dev/null +++ b/tests/milvus_benchmark/ci/jenkinsfile/cleanup.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge benchmark-test-${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy new file mode 100644 index 0000000000..f85ec8364b --- /dev/null +++ b/tests/milvus_benchmark/ci/jenkinsfile/deploy_test.groovy @@ -0,0 +1,20 @@ +timeout(time: 1440, unit: 'MINUTES') { + try { + dir ("milvus-helm") { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "${HELM_BRANCH}"]], userRemoteConfigs: [[url: "${HELM_URL}", name: 'origin', refspec: "+refs/heads/${HELM_BRANCH}:refs/remotes/origin/${HELM_BRANCH}"]]]) + } + dir ("milvus_benchmark") { + print "Git clone url: ${TEST_URL}:${TEST_BRANCH}" + checkout([$class: 'GitSCM', branches: [[name: "${TEST_BRANCH}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "${TEST_URL}", name: 'origin', refspec: "+refs/heads/${TEST_BRANCH}:refs/remotes/origin/${TEST_BRANCH}"]]]) + print "Install requirements" + sh "python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com" + sh "python3 -m pip install git+${TEST_LIB_URL}" + sh "python3 main.py --hostname=${params.SERVER_HOST} --image-tag=${IMAGE_TAG} --image-type=${params.IMAGE_TYPE} --suite=suites/${params.SUITE}" + } + } catch (exc) { + echo 'Deploy Test Failed !' + throw exc + } +} diff --git a/tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy b/tests/milvus_benchmark/ci/jenkinsfile/notify.groovy similarity index 100% rename from tests/milvus_ann_acc/ci/jenkinsfile/notify.groovy rename to tests/milvus_benchmark/ci/jenkinsfile/notify.groovy diff --git a/tests/milvus_ann_acc/ci/main_jenkinsfile b/tests/milvus_benchmark/ci/main_jenkinsfile similarity index 66% rename from tests/milvus_ann_acc/ci/main_jenkinsfile rename to tests/milvus_benchmark/ci/main_jenkinsfile index 9fdac4fc6e..08b987eb4a 100644 --- a/tests/milvus_ann_acc/ci/main_jenkinsfile +++ b/tests/milvus_benchmark/ci/main_jenkinsfile @@ -6,9 +6,10 @@ pipeline { } parameters{ - choice choices: ['cpu', 'gpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE' - string defaultValue: '0.6.0', description: 'server image version', name: 'IMAGE_VERSION', trim: true - string defaultValue: 'suite.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true + choice choices: ['gpu', 'cpu'], description: 'cpu or gpu version', name: 'IMAGE_TYPE' + string defaultValue: 'master', description: 'server image version', name: 'IMAGE_VERSION', trim: true + choice choices: ['poseidon', 'eros', 'apollo', 'athena'], description: 'server host', name: 'SERVER_HOST' + string defaultValue: 'gpu_search_performance_sift1m.yaml', description: 'test suite config yaml', name: 'SUITE', trim: true string defaultValue: '09509e53-9125-4f5d-9ce8-42855987ad67', description: 'git credentials', name: 'GIT_USER', trim: true } @@ -16,15 +17,16 @@ pipeline { IMAGE_TAG = "${params.IMAGE_VERSION}-${params.IMAGE_TYPE}-ubuntu18.04-release" HELM_URL = "https://github.com/milvus-io/milvus-helm.git" HELM_BRANCH = "0.6.0" - TEST_URL = "git@192.168.1.105:Test/milvus_ann_acc.git" - TEST_BRANCH = "0.6.0" + TEST_URL = "git@192.168.1.105:Test/milvus_benchmark.git" + TEST_BRANCH = "master" + TEST_LIB_URL = "http://192.168.1.105:6060/Test/milvus_metrics.git" } stages { stage("Setup env") { agent { kubernetes { - label 'dev-test' + label "test-benchmark-${env.JOB_NAME}-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -44,14 +46,26 @@ pipeline { - name: kubeconf mountPath: /root/.kube/ readOnly: true - - name: hdf5-path - mountPath: /test + - name: raw-data-path + mountPath: /poc readOnly: true + - name: db-data-path + mountPath: /test + readOnly: false volumes: - name: kubeconf secret: secretName: test-cluster-config - - name: hdf5-path + - name: raw-data-path + flexVolume: + driver: "fstab/cifs" + fsType: "cifs" + secretRef: + name: "cifs-test-secret" + options: + networkPath: "//192.168.1.126/poc" + mountOptions: "vers=1.0" + - name: db-data-path flexVolume: driver: "fstab/cifs" fsType: "cifs" @@ -65,30 +79,19 @@ pipeline { } stages { - stage("Deploy Default Server") { + stage("Deploy Test") { steps { - gitlabCommitStatus(name: 'Accuracy Test') { + gitlabCommitStatus(name: 'Deploy Test') { container('milvus-testframework') { script { - print "In Deploy Default Server Stage" - load "${env.WORKSPACE}/ci/jenkinsfile/deploy_default_server.groovy" - } - } - } - } - } - stage("Acc Test") { - steps { - gitlabCommitStatus(name: 'Accuracy Test') { - container('milvus-testframework') { - script { - print "In Acc test stage" - load "${env.WORKSPACE}/ci/jenkinsfile/acc_test.groovy" + print "In Deploy Test Stage" + load "${env.WORKSPACE}/ci/jenkinsfile/deploy_test.groovy" } } } } } + stage ("Cleanup Env") { steps { gitlabCommitStatus(name: 'Cleanup Env') { @@ -111,17 +114,17 @@ pipeline { } success { script { - echo "Milvus ann-accuracy test success !" + echo "Milvus benchmark test success !" } } aborted { script { - echo "Milvus ann-accuracy test aborted !" + echo "Milvus benchmark test aborted !" } } failure { script { - echo "Milvus ann-accuracy test failed !" + echo "Milvus benchmark test failed !" } } } diff --git a/tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml b/tests/milvus_benchmark/ci/pod_containers/milvus-testframework.yaml similarity index 100% rename from tests/milvus_ann_acc/ci/pod_containers/milvus-testframework.yaml rename to tests/milvus_benchmark/ci/pod_containers/milvus-testframework.yaml diff --git a/tests/milvus_benchmark/client.py b/tests/milvus_benchmark/client.py index 608d760813..62e3263ecd 100644 --- a/tests/milvus_benchmark/client.py +++ b/tests/milvus_benchmark/client.py @@ -1,8 +1,8 @@ +import sys import pdb import random import logging import json -import sys import time, datetime from multiprocessing import Process from milvus import Milvus, IndexType, MetricType @@ -12,7 +12,14 @@ logger = logging.getLogger("milvus_benchmark.client") SERVER_HOST_DEFAULT = "127.0.0.1" # SERVER_HOST_DEFAULT = "192.168.1.130" SERVER_PORT_DEFAULT = 19530 - +INDEX_MAP = { + "flat": IndexType.FLAT, + "ivf_flat": IndexType.IVFLAT, + "ivf_sq8": IndexType.IVF_SQ8, + "nsg": IndexType.RNSG, + "ivf_sq8h": IndexType.IVF_SQ8H, + "ivf_pq": IndexType.IVF_PQ +} def time_wrapper(func): """ @@ -28,18 +35,30 @@ def time_wrapper(func): class MilvusClient(object): - def __init__(self, table_name=None, ip=None, port=None): + def __init__(self, table_name=None, ip=None, port=None, timeout=60): self._milvus = Milvus() self._table_name = table_name try: + i = 1 + start_time = time.time() if not ip: self._milvus.connect( host = SERVER_HOST_DEFAULT, port = SERVER_PORT_DEFAULT) else: - self._milvus.connect( - host = ip, - port = port) + # retry connect for remote server + while time.time() < start_time + timeout: + try: + self._milvus.connect( + host = ip, + port = port) + if self._milvus.connected() is True: + logger.debug("Try connect times: %d, %s" % (i, round(time.time() - start_time, 2))) + break + except Exception as e: + logger.debug("Milvus connect failed") + i = i + 1 + except Exception as e: raise e @@ -49,7 +68,7 @@ class MilvusClient(object): def check_status(self, status): if not status.OK(): logger.error(status.message) - raise Exception("Status not ok") + # raise Exception("Status not ok") def create_table(self, table_name, dimension, index_file_size, metric_type): if not self._table_name: @@ -58,6 +77,10 @@ class MilvusClient(object): metric_type = MetricType.L2 elif metric_type == "ip": metric_type = MetricType.IP + elif metric_type == "jaccard": + metric_type = MetricType.JACCARD + elif metric_type == "hamming": + metric_type = MetricType.HAMMING else: logger.error("Not supported metric_type: %s" % metric_type) create_param = {'table_name': table_name, @@ -75,20 +98,8 @@ class MilvusClient(object): @time_wrapper def create_index(self, index_type, nlist): - if index_type == "flat": - index_type = IndexType.FLAT - elif index_type == "ivf_flat": - index_type = IndexType.IVFLAT - elif index_type == "ivf_sq8": - index_type = IndexType.IVF_SQ8 - elif index_type == "nsg": - index_type = IndexType.NSG - elif index_type == "ivf_sq8h": - index_type = IndexType.IVF_SQ8H - elif index_type == "ivf_pq": - index_type = IndexType.IVF_PQ index_params = { - "index_type": index_type, + "index_type": INDEX_MAP[index_type], "nlist": nlist, } logger.info("Building index start, table_name: %s, index_params: %s" % (self._table_name, json.dumps(index_params))) @@ -96,7 +107,18 @@ class MilvusClient(object): self.check_status(status) def describe_index(self): - return self._milvus.describe_index(self._table_name) + status, result = self._milvus.describe_index(self._table_name) + index_type = None + for k, v in INDEX_MAP.items(): + if result._index_type == v: + index_type = k + break + nlist = result._nlist + res = { + "index_type": index_type, + "nlist": nlist + } + return res def drop_index(self): logger.info("Drop index: %s" % self._table_name) @@ -106,7 +128,7 @@ class MilvusClient(object): def query(self, X, top_k, nprobe): status, result = self._milvus.search_vectors(self._table_name, top_k, nprobe, X) self.check_status(status) - return status, result + return result def count(self): return self._milvus.get_table_row_count(self._table_name)[1] @@ -122,7 +144,7 @@ class MilvusClient(object): continue else: break - if i < timeout: + if i >= timeout: logger.error("Delete table timeout") def describe(self): @@ -131,8 +153,10 @@ class MilvusClient(object): def show_tables(self): return self._milvus.show_tables() - def exists_table(self): - status, res = self._milvus.has_table(self._table_name) + def exists_table(self, table_name=None): + if table_name is None: + table_name = self._table_name + status, res = self._milvus.has_table(table_name) self.check_status(status) return res @@ -140,6 +164,33 @@ class MilvusClient(object): def preload_table(self): return self._milvus.preload_table(self._table_name, timeout=3000) + def get_server_version(self): + status, res = self._milvus.server_version() + return res + + def get_server_mode(self): + return self.cmd("mode") + + def get_server_commit(self): + return self.cmd("build_commit_id") + + def get_server_config(self): + return json.loads(self.cmd("get_config *")) + + def get_mem_info(self): + result = json.loads(self.cmd("get_system_info")) + result_human = { + # unit: Gb + "memory_used": round(int(result["memory_used"]) / (1024*1024*1024), 2) + } + return result_human + + def cmd(self, command): + status, res = self._milvus._cmd(command) + logger.info("Server command: %s, result: %s" % (command, res)) + self.check_status(status) + return res + def fit(table_name, X): milvus = Milvus() @@ -265,6 +316,12 @@ if __name__ == "__main__": # data = mmap_fvecs("/poc/deep1b/deep1B_queries.fvecs") # data = sklearn.preprocessing.normalize(data, axis=1, norm='l2') # np.save("/test/milvus/deep1b/query.npy", data) + dimension = 4096 + insert_xb = 10000 + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + data = sklearn.preprocessing.normalize(insert_vectors, axis=1, norm='l2') + np.save("/test/milvus/raw_data/random/query_%d.npy" % dimension, data) + sys.exit() total_size = 100000000 # total_size = 1000000000 diff --git a/tests/milvus_benchmark/demo.py b/tests/milvus_benchmark/demo.py deleted file mode 100644 index 27152e0980..0000000000 --- a/tests/milvus_benchmark/demo.py +++ /dev/null @@ -1,51 +0,0 @@ -import os -import logging -import pdb -import time -import random -from multiprocessing import Process -import numpy as np -from client import MilvusClient - -nq = 100000 -dimension = 128 -run_count = 1 -table_name = "sift_10m_1024_128_ip" -insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] - -def do_query(milvus, table_name, top_ks, nqs, nprobe, run_count): - bi_res = [] - for index, nq in enumerate(nqs): - tmp_res = [] - for top_k in top_ks: - avg_query_time = 0.0 - total_query_time = 0.0 - vectors = insert_vectors[0:nq] - for i in range(run_count): - start_time = time.time() - status, query_res = milvus.query(vectors, top_k, nprobe) - total_query_time = total_query_time + (time.time() - start_time) - if status.code: - print(status.message) - avg_query_time = round(total_query_time / run_count, 2) - tmp_res.append(avg_query_time) - bi_res.append(tmp_res) - return bi_res - -while 1: - milvus_instance = MilvusClient(table_name, ip="192.168.1.197", port=19530) - top_ks = random.sample([x for x in range(1, 100)], 4) - nqs = random.sample([x for x in range(1, 1000)], 3) - nprobe = random.choice([x for x in range(1, 500)]) - res = do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) - status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) - if not status.OK(): - logger.error(status.message) - - # status = milvus_instance.drop_index() - if not status.OK(): - print(status.message) - index_type = "ivf_sq8" - status = milvus_instance.create_index(index_type, 16384) - if not status.OK(): - print(status.message) \ No newline at end of file diff --git a/tests/milvus_benchmark/docker_runner.py b/tests/milvus_benchmark/docker_runner.py index 7bedbf497c..78c9550df7 100644 --- a/tests/milvus_benchmark/docker_runner.py +++ b/tests/milvus_benchmark/docker_runner.py @@ -53,7 +53,6 @@ class DockerRunner(Runner): # milvus.create_index("ivf_sq8", 16384) res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) logger.info(res) - # wait for file merge time.sleep(table_size * dimension / 5000000) # Clear up @@ -71,7 +70,7 @@ class DockerRunner(Runner): container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) time.sleep(2) milvus = MilvusClient(table_name) - logger.debug(milvus._milvus.show_tables()) + logger.debug(milvus.show_tables()) # Check has table or not if not milvus.exists_table(): logger.warning("Table %s not existed, continue exec next params ..." % table_name) @@ -104,6 +103,92 @@ class DockerRunner(Runner): utils.print_table(headers, nqs, res) utils.remove_container(container) + elif run_type == "insert_performance": + for op_type, op_value in definition.items(): + # run docker mode + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + if not run_params: + logger.debug("No run params") + continue + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + table_name = param["table_name"] + volume_name = param["db_path_prefix"] + print(table_name) + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + for k, v in param.items(): + if k.startswith("server."): + # Update server config + utils.modify_config(k, v, type="server", db_slave=None) + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(table_name) + # Check has table or not + if milvus.exists_table(): + milvus.delete() + time.sleep(10) + milvus.create_table(table_name, dimension, index_file_size, metric_type) + # debug + # milvus.create_index("ivf_sq8", 16384) + res = self.do_insert(milvus, table_name, data_type, dimension, table_size, param["ni_per"]) + logger.info(res) + # wait for file merge + time.sleep(table_size * dimension / 5000000) + # Clear up + utils.remove_container(container) + + elif run_type == "search_performance": + for op_type, op_value in definition.items(): + # run docker mode + run_count = op_value["run_count"] + run_params = op_value["params"] + container = None + for index, param in enumerate(run_params): + logger.info("Definition param: %s" % str(param)) + table_name = param["dataset"] + volume_name = param["db_path_prefix"] + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + for k, v in param.items(): + if k.startswith("server."): + utils.modify_config(k, v, type="server") + container = utils.run_server(self.image, test_type="remote", volume_name=volume_name, db_slave=None) + time.sleep(2) + milvus = MilvusClient(table_name) + logger.debug(milvus.show_tables()) + # Check has table or not + if not milvus.exists_table(): + logger.warning("Table %s not existed, continue exec next params ..." % table_name) + continue + # parse index info + index_types = param["index.index_types"] + nlists = param["index.nlists"] + # parse top-k, nq, nprobe + top_ks, nqs, nprobes = parser.search_params_parser(param) + for index_type in index_types: + for nlist in nlists: + result = milvus.describe_index() + logger.info(result) + # milvus.drop_index() + # milvus.create_index(index_type, nlist) + result = milvus.describe_index() + logger.info(result) + logger.info(milvus.count()) + # preload index + milvus.preload_table() + logger.info("Start warm up query") + res = self.do_query(milvus, table_name, [1], [1], 1, 1) + logger.info("End warm up query") + # Run query test + for nprobe in nprobes: + logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) + res = self.do_query(milvus, table_name, top_ks, nqs, nprobe, run_count) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + utils.print_table(headers, nqs, res) + utils.remove_container(container) + elif run_type == "accuracy": """ { @@ -149,11 +234,9 @@ class DockerRunner(Runner): nlists = param["index.nlists"] # parse top-k, nq, nprobe top_ks, nqs, nprobes = parser.search_params_parser(param) - if sift_acc is True: # preload groundtruth data true_ids_all = self.get_groundtruth_ids(table_size) - acc_dict = {} for index_type in index_types: for nlist in nlists: diff --git a/tests/milvus_benchmark/k8s_runner.py b/tests/milvus_benchmark/k8s_runner.py new file mode 100644 index 0000000000..a8d35e63ea --- /dev/null +++ b/tests/milvus_benchmark/k8s_runner.py @@ -0,0 +1,473 @@ +import os +import logging +import pdb +import time +import re +import random +import traceback +from multiprocessing import Process +import numpy as np +from client import MilvusClient +import utils +import parser +from runner import Runner +from milvus_metrics.api import report +from milvus_metrics.models import Env, Hardware, Server, Metric +import utils + +logger = logging.getLogger("milvus_benchmark.k8s_runner") +namespace = "milvus" +DELETE_INTERVAL_TIME = 5 +# INSERT_INTERVAL = 100000 +INSERT_INTERVAL = 50000 +timestamp = int(time.time()) +default_path = "/var/lib/milvus" + +class K8sRunner(Runner): + """run docker mode""" + def __init__(self): + super(K8sRunner, self).__init__() + self.name = utils.get_unique_name() + self.host = None + self.ip = None + self.hostname = None + self.env_value = None + + def init_env(self, server_config, args): + self.hostname = args.hostname + # update server_config + helm_path = os.path.join(os.getcwd(), "../milvus-helm/milvus") + server_config_file = helm_path+"/ci/config/sqlite/%s/server_config.yaml" % (args.image_type) + if not os.path.exists(server_config_file): + raise Exception("File %s not existed" % server_config_file) + if server_config: + logger.debug("Update server config") + utils.update_server_config(server_config_file, server_config) + # update log_config + log_config_file = helm_path+"/config/log_config.conf" + if not os.path.exists(log_config_file): + raise Exception("File %s not existed" % log_config_file) + src_log_config_file = helm_path+"/config/log_config.conf.src" + if not os.path.exists(src_log_config_file): + # copy + os.system("cp %s %s" % (log_config_file, src_log_config_file)) + else: + # reset + os.system("cp %s %s" % (src_log_config_file, log_config_file)) + if "db_config.primary_path" in server_config: + os.system("sed -i 's#%s#%s#g' %s" % (default_path, server_config["db_config.primary_path"], log_config_file)) + + # with open(log_config_file, "r+") as fd: + # for line in fd.readlines(): + # fd.write(re.sub(r'^%s' % default_path, server_config["db_config.primary_path"], line)) + # update values + values_file_path = helm_path+"/values.yaml" + if not os.path.exists(values_file_path): + raise Exception("File %s not existed" % values_file_path) + utils.update_values(values_file_path, args.hostname) + try: + logger.debug("Start install server") + self.host, self.ip = utils.helm_install_server(helm_path, args.image_tag, args.image_type, self.name, namespace) + except Exception as e: + logger.error("Helm install server failed: %s" % str(e)) + logger.error(traceback.format_exc()) + self.clean_up() + return False + # for debugging + # self.host = "192.168.1.101" + if not self.host: + logger.error("Helm install server failed") + self.clean_up() + return False + return True + + def clean_up(self): + logger.debug(self.name) + utils.helm_del_server(self.name) + + def report_wrapper(self, milvus_instance, env_value, hostname, table_info, index_info, search_params): + metric = Metric() + metric.set_run_id(timestamp) + metric.env = Env(env_value) + metric.env.OMP_NUM_THREADS = 0 + metric.hardware = Hardware(name=hostname) + server_version = milvus_instance.get_server_version() + server_mode = milvus_instance.get_server_mode() + commit = milvus_instance.get_server_commit() + metric.server = Server(version=server_version, mode=server_mode, build_commit=commit) + metric.table = table_info + metric.index = index_info + metric.search = search_params + return metric + + def run(self, run_type, table): + logger.debug(run_type) + logger.debug(table) + table_name = table["table_name"] + milvus_instance = MilvusClient(table_name=table_name, ip=self.ip) + self.env_value = milvus_instance.get_server_config() + if run_type == "insert_performance": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + ni_per = table["ni_per"] + build_index = table["build_index"] + if milvus_instance.exists_table(): + milvus_instance.delete() + time.sleep(10) + index_info = {} + search_params = {} + milvus_instance.create_table(table_name, dimension, index_file_size, metric_type) + if build_index is True: + index_type = table["index_type"] + nlist = table["nlist"] + index_info = { + "index_type": index_type, + "index_nlist": nlist + } + milvus_instance.create_index(index_type, nlist) + res = self.do_insert(milvus_instance, table_name, data_type, dimension, table_size, ni_per) + logger.info(res) + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric.metrics = { + "type": "insert_performance", + "value": { + "total_time": res["total_time"], + "qps": res["qps"], + "ni_time": res["ni_time"] + } + } + report(metric) + logger.debug("Wait for file merge") + time.sleep(120) + + elif run_type == "build_performance": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + index_type = table["index_type"] + nlist = table["nlist"] + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + index_info = { + "index_type": index_type, + "index_nlist": nlist + } + if not milvus_instance.exists_table(): + logger.error("Table name: %s not existed" % table_name) + return + search_params = {} + start_time = time.time() + # drop index + logger.debug("Drop index") + milvus_instance.drop_index() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + milvus_instance.create_index(index_type, nlist) + logger.debug(milvus_instance.describe_index()) + end_time = time.time() + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric.metrics = { + "type": "build_performance", + "value": { + "build_time": round(end_time - start_time, 1), + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage + } + } + report(metric) + + elif run_type == "search_performance": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + run_count = table["run_count"] + search_params = table["search_params"] + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + if not milvus_instance.exists_table(): + logger.error("Table name: %s not existed" % table_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + index_info = { + "index_type": result["index_type"], + "index_nlist": result["nlist"] + } + logger.info(index_info) + nprobes = search_params["nprobes"] + top_ks = search_params["top_ks"] + nqs = search_params["nqs"] + milvus_instance.preload_table() + logger.info("Start warm up query") + res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) + logger.info("End warm up query") + for nprobe in nprobes: + logger.info("Search nprobe: %s" % nprobe) + res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) + headers = ["Nq/Top-k"] + headers.extend([str(top_k) for top_k in top_ks]) + utils.print_table(headers, nqs, res) + for index_nq, nq in enumerate(nqs): + for index_top_k, top_k in enumerate(top_ks): + search_param = { + "nprobe": nprobe, + "nq": nq, + "topk": top_k + } + search_time = res[index_nq][index_top_k] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric.metrics = { + "type": "search_performance", + "value": { + "search_time": search_time + } + } + report(metric) + + # for sift/deep datasets + elif run_type == "accuracy": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + search_params = table["search_params"] + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + if not milvus_instance.exists_table(): + logger.error("Table name: %s not existed" % table_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + index_info = { + "index_type": result["index_type"], + "index_nlist": result["nlist"] + } + logger.info(index_info) + nprobes = search_params["nprobes"] + top_ks = search_params["top_ks"] + nqs = search_params["nqs"] + milvus_instance.preload_table() + true_ids_all = self.get_groundtruth_ids(table_size) + for nprobe in nprobes: + logger.info("Search nprobe: %s" % nprobe) + for top_k in top_ks: + for nq in nqs: + total = 0 + search_param = { + "nprobe": nprobe, + "nq": nq, + "topk": top_k + } + result_ids, result_distances = self.do_query_ids(milvus_instance, table_name, top_k, nq, nprobe) + acc_value = self.get_recall_value(true_ids_all[:nq, :top_k].tolist(), result_ids) + logger.info("Query accuracy: %s" % acc_value) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric.metrics = { + "type": "accuracy", + "value": { + "acc": acc_value + } + } + report(metric) + + elif run_type == "ann_accuracy": + hdf5_source_file = table["source_file"] + table_name = table["table_name"] + index_file_sizes = table["index_file_sizes"] + index_types = table["index_types"] + nlists = table["nlists"] + search_params = table["search_params"] + nprobes = search_params["nprobes"] + top_ks = search_params["top_ks"] + nqs = search_params["nqs"] + data_type, dimension, metric_type = parser.parse_ann_table_name(table_name) + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + dataset = utils.get_dataset(hdf5_source_file) + if milvus_instance.exists_table(table_name): + logger.info("Re-create table: %s" % table_name) + milvus_instance.delete(table_name) + time.sleep(DELETE_INTERVAL_TIME) + true_ids = np.array(dataset["neighbors"]) + for index_file_size in index_file_sizes: + milvus_instance.create_table(table_name, dimension, index_file_size, metric_type) + logger.info(milvus_instance.describe()) + insert_vectors = self.normalize(metric_type, np.array(dataset["train"])) + # Insert batch once + # milvus_instance.insert(insert_vectors) + loops = len(insert_vectors) // INSERT_INTERVAL + 1 + for i in range(loops): + start = i*INSERT_INTERVAL + end = min((i+1)*INSERT_INTERVAL, len(insert_vectors)) + tmp_vectors = insert_vectors[start:end] + if start < end: + if not isinstance(tmp_vectors, list): + milvus_instance.insert(tmp_vectors.tolist(), ids=[i for i in range(start, end)]) + else: + milvus_instance.insert(tmp_vectors, ids=[i for i in range(start, end)]) + time.sleep(20) + logger.info("Table: %s, row count: %s" % (table_name, milvus_instance.count())) + if milvus_instance.count() != len(insert_vectors): + logger.error("Table row count is not equal to insert vectors") + return + for index_type in index_types: + for nlist in nlists: + milvus_instance.create_index(index_type, nlist) + # logger.info(milvus_instance.describe_index()) + logger.info("Start preload table: %s, index_type: %s, nlist: %s" % (table_name, index_type, nlist)) + milvus_instance.preload_table() + index_info = { + "index_type": index_type, + "index_nlist": nlist + } + for nprobe in nprobes: + for nq in nqs: + query_vectors = self.normalize(metric_type, np.array(dataset["test"][:nq])) + for top_k in top_ks: + search_params = { + "nq": len(query_vectors), + "nprobe": nprobe, + "topk": top_k + } + if not isinstance(query_vectors, list): + result = milvus_instance.query(query_vectors.tolist(), top_k, nprobe) + else: + result = milvus_instance.query(query_vectors, top_k, nprobe) + result_ids = result.id_array + acc_value = self.get_recall_value(true_ids[:nq, :top_k].tolist(), result_ids) + logger.info("Query ann_accuracy: %s" % acc_value) + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_params) + metric.metrics = { + "type": "ann_accuracy", + "value": { + "acc": acc_value + } + } + report(metric) + milvus_instance.delete() + + elif run_type == "search_stability": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + search_params = table["search_params"] + during_time = table["during_time"] + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + if not milvus_instance.exists_table(): + logger.error("Table name: %s not existed" % table_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + index_info = { + "index_type": result["index_type"], + "index_nlist": result["nlist"] + } + search_param = {} + logger.info(index_info) + g_nprobe = int(search_params["nprobes"].split("-")[1]) + g_top_k = int(search_params["top_ks"].split("-")[1]) + g_nq = int(search_params["nqs"].split("-")[1]) + l_nprobe = int(search_params["nprobes"].split("-")[0]) + l_top_k = int(search_params["top_ks"].split("-")[0]) + l_nq = int(search_params["nqs"].split("-")[0]) + milvus_instance.preload_table() + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + logger.debug(start_mem_usage) + logger.info("Start warm up query") + res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) + logger.info("End warm up query") + start_time = time.time() + while time.time() < start_time + during_time * 60: + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + nprobe = random.randint(l_nprobe, g_nprobe) + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k)) + result = milvus_instance.query(query_vectors, top_k, nprobe) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric.metrics = { + "type": "search_stability", + "value": { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage + } + } + report(metric) + + elif run_type == "stability": + (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) + search_params = table["search_params"] + insert_xb = table["insert_xb"] + insert_interval = table["insert_interval"] + during_time = table["during_time"] + table_info = { + "dimension": dimension, + "metric_type": metric_type, + "dataset_name": table_name + } + if not milvus_instance.exists_table(): + logger.error("Table name: %s not existed" % table_name) + return + logger.info(milvus_instance.count()) + result = milvus_instance.describe_index() + index_info = { + "index_type": result["index_type"], + "index_nlist": result["nlist"] + } + search_param = {} + logger.info(index_info) + g_nprobe = int(search_params["nprobes"].split("-")[1]) + g_top_k = int(search_params["top_ks"].split("-")[1]) + g_nq = int(search_params["nqs"].split("-")[1]) + l_nprobe = int(search_params["nprobes"].split("-")[0]) + l_top_k = int(search_params["top_ks"].split("-")[0]) + l_nq = int(search_params["nqs"].split("-")[0]) + milvus_instance.preload_table() + logger.info("Start warm up query") + res = self.do_query(milvus_instance, table_name, [1], [1], 1, 2) + logger.info("End warm up query") + start_mem_usage = milvus_instance.get_mem_info()["memory_used"] + start_row_count = milvus_instance.count() + start_time = time.time() + i = 0 + while time.time() < start_time + during_time * 60: + i = i + 1 + for j in range(insert_interval): + top_k = random.randint(l_top_k, g_top_k) + nq = random.randint(l_nq, g_nq) + nprobe = random.randint(l_nprobe, g_nprobe) + query_vectors = [[random.random() for _ in range(dimension)] for _ in range(nq)] + logger.debug("Query nprobe:%d, nq:%d, top-k:%d" % (nprobe, nq, top_k)) + result = milvus_instance.query(query_vectors, top_k, nprobe) + insert_vectors = [[random.random() for _ in range(dimension)] for _ in range(insert_xb)] + status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) + logger.debug("%d, row_count: %d" % (i, milvus_instance.count())) + end_mem_usage = milvus_instance.get_mem_info()["memory_used"] + end_row_count = milvus_instance.count() + metric = self.report_wrapper(milvus_instance, self.env_value, self.hostname, table_info, index_info, search_param) + metric.metrics = { + "type": "stability", + "value": { + "during_time": during_time, + "start_mem_usage": start_mem_usage, + "end_mem_usage": end_mem_usage, + "diff_mem": end_mem_usage - start_mem_usage, + "row_count_increments": end_row_count - start_row_count + } + } + report(metric) \ No newline at end of file diff --git a/tests/milvus_benchmark/local_runner.py b/tests/milvus_benchmark/local_runner.py index 22390e5470..c7822fb7bc 100644 --- a/tests/milvus_benchmark/local_runner.py +++ b/tests/milvus_benchmark/local_runner.py @@ -48,6 +48,8 @@ class LocalRunner(Runner): milvus = MilvusClient(table_name, ip=self.ip, port=self.port) logger.info(milvus.describe()) logger.info(milvus.describe_index()) + logger.info(milvus.count()) + logger.info(milvus.show_tables()) # parse index info index_types = param["index.index_types"] nlists = param["index.nlists"] @@ -64,7 +66,7 @@ class LocalRunner(Runner): logger.info("End preloading table") # Run query test logger.info("Start warm up query") - res = self.do_query(milvus, table_name, [1], [1], 1, 1) + res = self.do_query(milvus, table_name, [1], [1], 1, 2) logger.info("End warm up query") for nprobe in nprobes: logger.info("index_type: %s, nlist: %s, metric_type: %s, nprobe: %s" % (index_type, nlist, metric_type, nprobe)) @@ -204,12 +206,14 @@ class LocalRunner(Runner): # p.join() i = i + 1 milvus_instance = MilvusClient(table_name, ip=self.ip, port=self.port) - top_ks = random.sample([x for x in range(1, 100)], 4) - nqs = random.sample([x for x in range(1, 200)], 3) + top_ks = random.sample([x for x in range(1, 100)], 1) + nqs = random.sample([x for x in range(1, 200)], 2) nprobe = random.choice([x for x in range(1, 100)]) res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) # milvus_instance = MilvusClient(table_name) status, res = milvus_instance.insert(insert_vectors, ids=[x for x in range(len(insert_vectors))]) if not status.OK(): logger.error(status.message) + logger.debug(milvus.count()) + res = self.do_query(milvus_instance, table_name, top_ks, nqs, nprobe, run_count) # status = milvus_instance.create_index(index_type, 16384) diff --git a/tests/milvus_benchmark/main.py b/tests/milvus_benchmark/main.py index c11237c5e7..0af4a9ee81 100644 --- a/tests/milvus_benchmark/main.py +++ b/tests/milvus_benchmark/main.py @@ -4,26 +4,31 @@ import time import pdb import argparse import logging -import utils -from yaml import load, dump +import traceback from logging import handlers + +from yaml import full_load, dump from parser import operations_parser from local_runner import LocalRunner from docker_runner import DockerRunner +from k8s_runner import K8sRunner DEFAULT_IMAGE = "milvusdb/milvus:latest" -LOG_FOLDER = "benchmark_logs" -logger = logging.getLogger("milvus_benchmark") +LOG_FOLDER = "logs" -formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') -if not os.path.exists(LOG_FOLDER): - os.system('mkdir -p %s' % LOG_FOLDER) -fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10) -fileTimeHandler.suffix = "%Y%m%d.log" -fileTimeHandler.setFormatter(formatter) -logging.basicConfig(level=logging.DEBUG) -fileTimeHandler.setFormatter(formatter) -logger.addHandler(fileTimeHandler) +# formatter = logging.Formatter('[%(asctime)s] [%(levelname)-4s] [%(pathname)s:%(lineno)d] %(message)s') +# if not os.path.exists(LOG_FOLDER): +# os.system('mkdir -p %s' % LOG_FOLDER) +# fileTimeHandler = handlers.TimedRotatingFileHandler(os.path.join(LOG_FOLDER, 'milvus_benchmark'), "D", 1, 10) +# fileTimeHandler.suffix = "%Y%m%d.log" +# fileTimeHandler.setFormatter(formatter) +# logging.basicConfig(level=logging.DEBUG) +# fileTimeHandler.setFormatter(formatter) +# logger.addHandler(fileTimeHandler) +logging.basicConfig(format='%(asctime)s,%(msecs)d %(levelname)-8s [%(filename)s:%(lineno)d] %(message)s', + datefmt='%Y-%m-%d:%H:%M:%S', + level=logging.DEBUG) +logger = logging.getLogger("milvus_benchmark") def positive_int(s): @@ -51,30 +56,39 @@ def main(): parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '--image', - help='use the given image') + "--hostname", + default="eros", + help="server host name") + parser.add_argument( + "--image-tag", + default="", + help="image tag") + parser.add_argument( + "--image-type", + default="", + help="image type") + # parser.add_argument( + # "--run-count", + # default=1, + # type=positive_int, + # help="run times for each test") + # # performance / stability / accuracy test + # parser.add_argument( + # "--run-type", + # default="search_performance", + # help="run type, default performance") + parser.add_argument( + '--suite', + metavar='FILE', + help='load test suite from FILE', + default='suites/suite.yaml') parser.add_argument( '--local', action='store_true', help='use local milvus server') parser.add_argument( - "--run-count", - default=1, - type=positive_int, - help="run each db operation times") - # performance / stability / accuracy test - parser.add_argument( - "--run-type", - default="performance", - help="run type, default performance") - parser.add_argument( - '--suites', - metavar='FILE', - help='load test suites from FILE', - default='suites.yaml') - parser.add_argument( - '--ip', - help='server ip param for local mode', + '--host', + help='server host ip param for local mode', default='127.0.0.1') parser.add_argument( '--port', @@ -83,43 +97,60 @@ def main(): args = parser.parse_args() - operations = None # Get all benchmark test suites - if args.suites: - with open(args.suites) as f: - suites_dict = load(f) + if args.suite: + with open(args.suite) as f: + suite_dict = full_load(f) f.close() # With definition order - operations = operations_parser(suites_dict, run_type=args.run_type) + run_type, run_params = operations_parser(suite_dict) # init_env() - run_params = {"run_count": args.run_count} + # run_params = {"run_count": args.run_count} - if args.image: + if args.image_tag: + namespace = "milvus" + logger.debug(args) # for docker mode if args.local: - logger.error("Local mode and docker mode are incompatible arguments") + logger.error("Local mode and docker mode are incompatible") sys.exit(-1) # Docker pull image - if not utils.pull_image(args.image): - raise Exception('Image %s pull failed' % image) - + # if not utils.pull_image(args.image): + # raise Exception('Image %s pull failed' % image) # TODO: Check milvus server port is available - logger.info("Init: remove all containers created with image: %s" % args.image) - utils.remove_all_containers(args.image) - runner = DockerRunner(args.image) - for operation_type in operations: - logger.info("Start run test, test type: %s" % operation_type) - run_params["params"] = operations[operation_type] - runner.run({operation_type: run_params}, run_type=args.run_type) - logger.info("Run params: %s" % str(run_params)) + # logger.info("Init: remove all containers created with image: %s" % args.image) + # utils.remove_all_containers(args.image) + # runner = DockerRunner(args) + tables = run_params["tables"] + for table in tables: + # run tests + server_config = table["server"] + logger.debug(server_config) + runner = K8sRunner() + if runner.init_env(server_config, args): + logger.debug("Start run tests") + try: + runner.run(run_type, table) + except Exception as e: + logger.error(str(e)) + logger.error(traceback.format_exc()) + finally: + runner.clean_up() + else: + logger.error("Runner init failed") + # for operation_type in operations: + # logger.info("Start run test, test type: %s" % operation_type) + # run_params["params"] = operations[operation_type] + # runner.run({operation_type: run_params}, run_type=args.run_type) + # logger.info("Run params: %s" % str(run_params)) if args.local: # for local mode - ip = args.ip + host = args.host port = args.port - runner = LocalRunner(ip, port) + runner = LocalRunner(host, port) for operation_type in operations: logger.info("Start run local mode test, test type: %s" % operation_type) run_params["params"] = operations[operation_type] diff --git a/tests/milvus_benchmark/parser.py b/tests/milvus_benchmark/parser.py index 1c1d2605f2..4727071e9b 100644 --- a/tests/milvus_benchmark/parser.py +++ b/tests/milvus_benchmark/parser.py @@ -4,9 +4,12 @@ import logging logger = logging.getLogger("milvus_benchmark.parser") -def operations_parser(operations, run_type="performance"): - definitions = operations[run_type] - return definitions +def operations_parser(operations): + if not operations: + raise Exception("No operations in suite defined") + for run_type, run_params in operations.items(): + logger.debug(run_type) + return (run_type, run_params) def table_parser(table_name): @@ -26,6 +29,23 @@ def table_parser(table_name): return (data_type, table_size, index_file_size, dimension, metric_type) +def parse_ann_table_name(table_name): + data_type = table_name.split("_")[0] + dimension = int(table_name.split("_")[1]) + metric = table_name.split("_")[-1] + # metric = table_name.attrs['distance'] + # dimension = len(table_name["train"][0]) + if metric == "euclidean": + metric_type = "l2" + elif metric == "angular": + metric_type = "ip" + elif metric == "jaccard": + metric_type = "jaccard" + elif metric == "hamming": + metric_type = "hamming" + return ("ann_"+data_type, dimension, metric_type) + + def search_params_parser(param): # parse top-k, set default value if top-k not in param if "top_ks" not in param: diff --git a/tests/milvus_benchmark/requirements.txt b/tests/milvus_benchmark/requirements.txt index ee6a8a11ff..cd459f296d 100644 --- a/tests/milvus_benchmark/requirements.txt +++ b/tests/milvus_benchmark/requirements.txt @@ -2,8 +2,9 @@ numpy==1.16.3 pymilvus-test>=0.2.0 scikit-learn==0.19.1 h5py==2.7.1 -influxdb==5.2.2 +# influxdb==5.2.2 pyyaml>=5.1 tableprint==0.8.0 ansicolors==1.1.8 scipy==1.3.1 +kubernetes==10.0.1 diff --git a/tests/milvus_benchmark/runner.py b/tests/milvus_benchmark/runner.py index 76851fa69c..8d1cc61778 100644 --- a/tests/milvus_benchmark/runner.py +++ b/tests/milvus_benchmark/runner.py @@ -5,24 +5,26 @@ import time import random from multiprocessing import Process import numpy as np +import sklearn.preprocessing from client import MilvusClient import utils import parser logger = logging.getLogger("milvus_benchmark.runner") -SERVER_HOST_DEFAULT = "127.0.0.1" -SERVER_PORT_DEFAULT = 19530 VECTORS_PER_FILE = 1000000 SIFT_VECTORS_PER_FILE = 100000 +JACCARD_VECTORS_PER_FILE = 2000000 + MAX_NQ = 10001 FILE_PREFIX = "binary_" # FOLDER_NAME = 'ann_1000m/source_data' -SRC_BINARY_DATA_DIR = '/poc/yuncong/yunfeng/random_data' -SRC_BINARY_DATA_DIR_high = '/test/milvus/raw_data/random' -SIFT_SRC_DATA_DIR = '/poc/yuncong/ann_1000m/' -SIFT_SRC_BINARY_DATA_DIR = SIFT_SRC_DATA_DIR + 'source_data' +SRC_BINARY_DATA_DIR = '/test/milvus/raw_data/random/' +SIFT_SRC_DATA_DIR = '/test/milvus/raw_data/sift1b/' +DEEP_SRC_DATA_DIR = '/test/milvus/raw_data/deep1b/' +JACCARD_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' +HAMMING_SRC_DATA_DIR = '/test/milvus/raw_data/jaccard/' SIFT_SRC_GROUNDTRUTH_DATA_DIR = SIFT_SRC_DATA_DIR + 'gnd' WARM_TOP_K = 1 @@ -44,16 +46,19 @@ GROUNDTRUTH_MAP = { } -def gen_file_name(idx, table_dimension, data_type): +def gen_file_name(idx, dimension, data_type): s = "%05d" % idx - fname = FILE_PREFIX + str(table_dimension) + "d_" + s + ".npy" + fname = FILE_PREFIX + str(dimension) + "d_" + s + ".npy" if data_type == "random": - if table_dimension == 512: - fname = SRC_BINARY_DATA_DIR+'/'+fname - elif table_dimension >= 4096: - fname = SRC_BINARY_DATA_DIR_high+'/'+fname + fname = SRC_BINARY_DATA_DIR+fname elif data_type == "sift": - fname = SIFT_SRC_BINARY_DATA_DIR+'/'+fname + fname = SIFT_SRC_DATA_DIR+fname + elif data_type == "deep": + fname = DEEP_SRC_DATA_DIR+fname + elif data_type == "jaccard": + fname = JACCARD_SRC_DATA_DIR+fname + elif data_type == "hamming": + fname = HAMMING_SRC_DATA_DIR+fname return fname @@ -62,9 +67,15 @@ def get_vectors_from_binary(nq, dimension, data_type): if nq > MAX_NQ: raise Exception("Over size nq") if data_type == "random": - file_name = gen_file_name(0, dimension, data_type) + file_name = SRC_BINARY_DATA_DIR+'query_%d.npy' % dimension elif data_type == "sift": - file_name = SIFT_SRC_DATA_DIR+'/'+'query.npy' + file_name = SIFT_SRC_DATA_DIR+'query.npy' + elif data_type == "deep": + file_name = DEEP_SRC_DATA_DIR+'query.npy' + elif data_type == "jaccard": + file_name = JACCARD_SRC_DATA_DIR+'query.npy' + elif data_type == "hamming": + file_name = HAMMING_SRC_DATA_DIR+'query.npy' data = np.load(file_name) vectors = data[0:nq].tolist() return vectors @@ -74,6 +85,21 @@ class Runner(object): def __init__(self): pass + def normalize(self, metric_type, X): + if metric_type == "ip": + logger.info("Set normalize for metric_type: %s" % metric_type) + X = sklearn.preprocessing.normalize(X, axis=1, norm='l2') + X = X.astype(np.float32) + elif metric_type == "l2": + X = X.astype(np.float32) + elif metric_type == "jaccard" or metric_type == "hamming": + tmp = [] + for index, item in enumerate(X): + new_vector = bytes(np.packbits(item, axis=-1).tolist()) + tmp.append(new_vector) + X = tmp + return X + def do_insert(self, milvus, table_name, data_type, dimension, size, ni): ''' @params: @@ -101,14 +127,18 @@ class Runner(object): vectors_per_file = 10000 elif data_type == "sift": vectors_per_file = SIFT_VECTORS_PER_FILE + elif data_type == "jaccard" or data_type == "hamming": + vectors_per_file = JACCARD_VECTORS_PER_FILE + else: + raise Exception("data_type: %s not supported" % data_type) if size % vectors_per_file or ni > vectors_per_file: raise Exception("Not invalid table size or ni") file_num = size // vectors_per_file for i in range(file_num): file_name = gen_file_name(i, dimension, data_type) - logger.info("Load npy file: %s start" % file_name) + # logger.info("Load npy file: %s start" % file_name) data = np.load(file_name) - logger.info("Load npy file: %s end" % file_name) + # logger.info("Load npy file: %s end" % file_name) loops = vectors_per_file // ni for j in range(loops): vectors = data[j*ni:(j+1)*ni].tolist() @@ -130,28 +160,26 @@ class Runner(object): bi_res["ni_time"] = ni_time return bi_res - def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count): + def do_query(self, milvus, table_name, top_ks, nqs, nprobe, run_count=1): + bi_res = [] (data_type, table_size, index_file_size, dimension, metric_type) = parser.table_parser(table_name) base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) - - bi_res = [] - for index, nq in enumerate(nqs): + for nq in nqs: tmp_res = [] vectors = base_query_vectors[0:nq] for top_k in top_ks: avg_query_time = 0.0 - total_query_time = 0.0 + min_query_time = 0.0 logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) for i in range(run_count): logger.info("Start run query, run %d of %s" % (i+1, run_count)) start_time = time.time() - status, query_res = milvus.query(vectors, top_k, nprobe) - total_query_time = total_query_time + (time.time() - start_time) - if status.code: - logger.error("Query failed with message: %s" % status.message) - avg_query_time = round(total_query_time / run_count, 2) - logger.info("Avarage query time: %.2f" % avg_query_time) - tmp_res.append(avg_query_time) + query_res = milvus.query(vectors, top_k, nprobe) + interval_time = time.time() - start_time + if (i == 0) or (min_query_time > interval_time): + min_query_time = interval_time + logger.info("Min query time: %.2f" % min_query_time) + tmp_res.append(round(min_query_time, 2)) bi_res.append(tmp_res) return bi_res @@ -160,10 +188,7 @@ class Runner(object): base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) vectors = base_query_vectors[0:nq] logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) - status, query_res = milvus.query(vectors, top_k, nprobe) - if not status.OK(): - msg = "Query failed with message: %s" % status.message - raise Exception(msg) + query_res = milvus.query(vectors, top_k, nprobe) result_ids = [] result_distances = [] for result in query_res: @@ -181,10 +206,7 @@ class Runner(object): base_query_vectors = get_vectors_from_binary(MAX_NQ, dimension, data_type) vectors = base_query_vectors[0:nq] logger.info("Start query, query params: top-k: {}, nq: {}, actually length of vectors: {}".format(top_k, nq, len(vectors))) - status, query_res = milvus.query(vectors, top_k, nprobe) - if not status.OK(): - msg = "Query failed with message: %s" % status.message - raise Exception(msg) + query_res = milvus.query(vectors, top_k, nprobe) # if file existed, cover it if os.path.isfile(id_store_name): os.remove(id_store_name) @@ -212,15 +234,17 @@ class Runner(object): # get the accuracy return self.get_recall_value(flat_id_list, index_id_list) - def get_recall_value(self, flat_id_list, index_id_list): + def get_recall_value(self, true_ids, result_ids): """ Use the intersection length """ sum_radio = 0.0 - for index, item in enumerate(index_id_list): - tmp = set(item).intersection(set(flat_id_list[index])) + for index, item in enumerate(result_ids): + # tmp = set(item).intersection(set(flat_id_list[index])) + tmp = set(true_ids[index]).intersection(set(item)) sum_radio = sum_radio + len(tmp) / len(item) - return round(sum_radio / len(index_id_list), 3) + # logger.debug(sum_radio) + return round(sum_radio / len(result_ids), 3) """ Implementation based on: diff --git a/tests/milvus_benchmark/scripts/default_config.json b/tests/milvus_benchmark/scripts/default_config.json new file mode 100644 index 0000000000..f63b3d746b --- /dev/null +++ b/tests/milvus_benchmark/scripts/default_config.json @@ -0,0 +1,86 @@ +[ + { + "job_name": "milvus-apollo", + "build_params": { + "SUITE": "cpu_accuracy_ann.yaml", + "IMAGE_TYPE": "cpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "apollo" + } + }, + { + "job_name": "milvus-apollo", + "build_params": { + "SUITE": "cpu_stability_sift50m.yaml", + "IMAGE_TYPE": "cpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "apollo" + } + }, + + { + "job_name": "milvus-eros", + "build_params": { + "SUITE": "gpu_accuracy_ann.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "eros" + } + }, + { + "job_name": "milvus-eros", + "build_params": { + "SUITE": "gpu_search_stability.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "eros" + } + }, + { + "job_name": "milvus-eros", + "build_params": { + "SUITE": "gpu_build_performance.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "eros" + } + }, + + { + "job_name": "milvus-poseidon", + "build_params": { + "SUITE": "gpu_search_performance.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "poseidon" + } + }, + { + "job_name": "milvus-poseidon", + "build_params": { + "SUITE": "cpu_search_performance.yaml", + "IMAGE_TYPE": "cpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "poseidon" + } + }, + { + "job_name": "milvus-poseidon", + "build_params": { + "SUITE": "insert_performance.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "poseidon" + } + }, + { + "job_name": "milvus-poseidon", + "build_params": { + "SUITE": "gpu_accuracy.yaml", + "IMAGE_TYPE": "gpu", + "IMAGE_VERSION": "master", + "SERVER_HOST": "poseidon" + } + } + +] \ No newline at end of file diff --git a/tests/milvus_benchmark/scripts/scheduler.py b/tests/milvus_benchmark/scripts/scheduler.py new file mode 100644 index 0000000000..613397a8c5 --- /dev/null +++ b/tests/milvus_benchmark/scripts/scheduler.py @@ -0,0 +1,28 @@ +import json +from pprint import pprint +import jenkins + +JENKINS_URL = "****" +server = jenkins.Jenkins(JENKINS_URL, username='****', password='****') +user = server.get_whoami() +version = server.get_version() +print('Hello %s from Jenkins %s' % (user['fullName'], version)) + + +# print(job_config) +# build_params = { +# "SUITE": "gpu_accuracy_ann_debug.yaml", +# "IMAGE_TYPE": "cpu", +# "IMAGE_VERSION": "tanimoto_distance", +# "SERVER_HOST": "eros" +# } +# print(server.build_job(job_name, build_params)) + + +with open("default_config.json") as json_file: + data = json.load(json_file) + for config in data: + build_params = config["build_params"] + job_name = config["job_name"] + res = server.build_job(job_name, build_params) + print(job_name, res) diff --git a/tests/milvus_benchmark/suites.yaml b/tests/milvus_benchmark/suites.yaml deleted file mode 100644 index f30b963c03..0000000000 --- a/tests/milvus_benchmark/suites.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# data sets -datasets: - hf5: - gist-960,sift-128 - npy: - 50000000-512, 100000000-512 - -operations: - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including nprpbe/nlist/use_blas_threshold/.. - [ - # debug - # {"dataset": "ip_ivfsq8_1000", "top_ks": [16], "nqs": [1], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110}, - - {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110}, - {"dataset": "ip_ivfsq8_1000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110}, - {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110}, - {"dataset": "ip_ivfsq8_5000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110}, - {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512], "nqs": [1, 10, 100, 500, 800, 1000], "server.nprobe": 1, "server.use_blas_threshold": 800, "server.cpu_cache_capacity": 110}, - # {"dataset": "ip_ivfsq8_40000", "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], "nqs": [1, 10, 100, 1000], "server.nprobe": 10, "server.use_blas_threshold": 20, "server.cpu_cache_capacity": 110}, - ] - - # interface: add_vectors - insert: - # index_type: flat/ivf_flat/ivf_sq8 - [ - # debug - - - {"table_name": "ip_ivf_flat_20m_1024", "table.index_type": "ivf_flat", "server.index_building_threshold": 1024, "table.size": 20000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110}, - {"table_name": "ip_ivf_sq8_50m_1024", "table.index_type": "ivf_sq8", "server.index_building_threshold": 1024, "table.size": 50000000, "table.ni": 100000, "table.dim": 512, "server.cpu_cache_capacity": 110}, - ] - - # TODO: interface: build_index - build: [] - diff --git a/tests/milvus_benchmark/suites/cpu_accuracy.yaml b/tests/milvus_benchmark/suites/cpu_accuracy.yaml new file mode 100644 index 0000000000..2d71da9315 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy.yaml @@ -0,0 +1,119 @@ +accuracy: + tables: + # sift1b + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + + # sift50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 60 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + cache_config.cpu_cache_capacity: 100 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml new file mode 100644 index 0000000000..99553f0c9b --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_accuracy_ann.yaml @@ -0,0 +1,68 @@ +ann_accuracy: + tables: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + table_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5 + table_name: gist_960_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + table_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_pq', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_build_performance.yaml b/tests/milvus_benchmark/suites/cpu_build_performance.yaml new file mode 100644 index 0000000000..98f7014375 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_build_performance.yaml @@ -0,0 +1,36 @@ +build_performance: + tables: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 6 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # index_type: ivf_sq8 + # nlist: 4096 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + index_type: ivf_sq8 + nlist: 8192 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/cpu_search_performance.yaml b/tests/milvus_benchmark/suites/cpu_search_performance.yaml new file mode 100644 index 0000000000..d971579756 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_performance.yaml @@ -0,0 +1,169 @@ +search_performance: + tables: + # sift_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + + # random_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf + cache_config.cpu_cache_capacity: 110 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg + cache_config.cpu_cache_capacity: 200 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + + # sift_1b + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/cpu_search_stability.yaml b/tests/milvus_benchmark/suites/cpu_search_stability.yaml new file mode 100644 index 0000000000..85df288ad0 --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_search_stability.yaml @@ -0,0 +1,20 @@ +search_stability: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + during_time: 240 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 diff --git a/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml new file mode 100644 index 0000000000..fd79a49d7c --- /dev/null +++ b/tests/milvus_benchmark/suites/cpu_stability_sift50m.yaml @@ -0,0 +1,27 @@ +stability: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192_stability + cache_config.cpu_cache_capacity: 64 + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + during_time: 480 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 100000 + # insert after search 4 times + insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy.yaml b/tests/milvus_benchmark/suites/gpu_accuracy.yaml new file mode 100644 index 0000000000..2791851b1b --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy.yaml @@ -0,0 +1,157 @@ +accuracy: + tables: + # sift1b + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_1b_2048_128_l2 + # search_params: + # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + # top_ks: [64] + # nqs: [1000] + + # sift50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 60 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # search_params: + # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + # top_ks: [64] + # nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + cache_config.cpu_cache_capacity: 100 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml new file mode 100644 index 0000000000..00cdb388fb --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_ann.yaml @@ -0,0 +1,68 @@ +ann_accuracy: + tables: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 + table_name: sift_128_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/gist-960-euclidean.hdf5 + table_name: gist_960_euclidean + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'ivf_pq', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 + table_name: glove_200_angular + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat', 'ivf_sq8', 'ivf_sq8h', 'nsg'] + nlists: [16384] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml new file mode 100644 index 0000000000..b9d2e22b3d --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_ann_debug.yaml @@ -0,0 +1,47 @@ +ann_accuracy: + tables: + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/kosarak-27983-jaccard.hdf5 + table_name: kosarak_27984_jaccard + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + nlists: [2048] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [10] + nqs: [10000] + + - + server: + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + source_file: /test/milvus/ann_hdf5/sift-256-hamming.hdf5 + table_name: sift_256_hamming + index_file_sizes: [1024] + index_types: ['flat', 'ivf_flat'] + nlists: [2048] + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [100] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml new file mode 100644 index 0000000000..80d9e69bd1 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_sift1b.yaml @@ -0,0 +1,59 @@ +accuracy: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml new file mode 100644 index 0000000000..6982fa877d --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_sift1m.yaml @@ -0,0 +1,40 @@ +accuracy: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml new file mode 100644 index 0000000000..e065753011 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_accuracy_sift50m.yaml @@ -0,0 +1,80 @@ +accuracy: + tables: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # search_params: + # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + # top_ks: [64] + # nqs: [1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # search_params: + # nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + # top_ks: [64] + # nqs: [1000] + + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192] + top_ks: [64] + nqs: [1000] + + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + search_params: + nprobes: [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096] + top_ks: [64] + nqs: [1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance.yaml b/tests/milvus_benchmark/suites/gpu_build_performance.yaml new file mode 100644 index 0000000000..306d65d609 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_performance.yaml @@ -0,0 +1,36 @@ +build_performance: + tables: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: random_50m_1024_512_ip + # index_type: ivf_sq8h + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + index_type: ivf_sq8 + nlist: 4096 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml new file mode 100644 index 0000000000..89eaf8eb1f --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_performance_hamming50m.yaml @@ -0,0 +1,36 @@ +build_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: hamming_50m_128_512_hamming + index_type: ivf_flat + nlist: 2048 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: hamming_50m_128_512_hamming + index_type: flat + nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml new file mode 100644 index 0000000000..15adc2d718 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_build_performance_jaccard50m.yaml @@ -0,0 +1,36 @@ +build_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: jaccard_50m_128_512_jaccard + index_type: ivf_flat + nlist: 2048 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: jaccard_50m_128_512_jaccard + index_type: flat + nlist: 2048 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance.yaml b/tests/milvus_benchmark/suites/gpu_search_performance.yaml new file mode 100644 index 0000000000..161719ba57 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance.yaml @@ -0,0 +1,247 @@ +search_performance: + tables: + # sift_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + + # random_50m + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_ivf + cache_config.cpu_cache_capacity: 110 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h + cache_config.cpu_cache_capacity: 30 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg + cache_config.cpu_cache_capacity: 200 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 6 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: random_50m_1024_512_ip + run_count: 2 + search_params: + nprobes: [8] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + + # sift_1b + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + # cache_config.cpu_cache_capacity: 150 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_1b_2048_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml new file mode 100644 index 0000000000..b212c4eda1 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_hamming50m.yaml @@ -0,0 +1,22 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: hamming_50m_128_512_hamming + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml new file mode 100644 index 0000000000..e0d6957e3b --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_jaccard50m.yaml @@ -0,0 +1,22 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf + cache_config.cpu_cache_capacity: 32 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: jaccard_50m_128_512_jaccard + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml new file mode 100644 index 0000000000..534a82d501 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_random50m.yaml @@ -0,0 +1,82 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/random_50m_2048_512_ip_sq8 + cache_config.cpu_cache_capacity: 110 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: random_50m_2048_512_ip + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8 + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: random_50m_2048_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_sq8h + # cache_config.cpu_cache_capacity: 30 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/random_50m_1024_512_ip_nsg + # cache_config.cpu_cache_capacity: 200 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: random_50m_1024_512_ip + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml new file mode 100644 index 0000000000..88aac71806 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift1b.yaml @@ -0,0 +1,62 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8 + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_sq8h + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1b_2048_128_l2_pq + cache_config.cpu_cache_capacity: 150 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1b_2048_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml new file mode 100644 index 0000000000..1b2b8485c7 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift1m.yaml @@ -0,0 +1,42 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + run_count: 1 + search_params: + nprobes: [8, 32] + top_ks: [1, 16, 64, 128, 256, 512, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml new file mode 100644 index 0000000000..75d3e5f110 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_performance_sift50m.yaml @@ -0,0 +1,146 @@ +search_performance: + tables: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_ivf + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h + # cache_config.cpu_cache_capacity: 16 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # git issue num: #626 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_pq + # cache_config.cpu_cache_capacity: 32 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8, 32] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_nsg + # cache_config.cpu_cache_capacity: 50 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 200 + # gpu_resource_config.enable: true + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # - gpu1 + # gpu_resource_config.build_index_resources: + # - gpu0 + # - gpu1 + # table_name: sift_50m_1024_128_l2 + # run_count: 2 + # search_params: + # nprobes: [8] + # top_ks: [1, 10, 100, 1000] + # nqs: [1, 10, 100, 200, 500, 1000] + + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 16 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 200 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + run_count: 2 + search_params: + nprobes: [8, 32] + top_ks: [1, 10, 100, 1000] + nqs: [1, 10, 100, 200, 500, 1000] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/gpu_search_stability.yaml b/tests/milvus_benchmark/suites/gpu_search_stability.yaml new file mode 100644 index 0000000000..306537138c --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_search_stability.yaml @@ -0,0 +1,23 @@ +search_stability: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8 + cache_config.cpu_cache_capacity: 50 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + - gpu2 + - gpu3 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + during_time: 240 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 diff --git a/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml b/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml new file mode 100644 index 0000000000..9d4c75fb99 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_stability_sift1m.yaml @@ -0,0 +1,26 @@ +stability: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf_stability + cache_config.cpu_cache_capacity: 64 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + during_time: 10 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 10000 + # insert after search 3 times + insert_interval: 3 diff --git a/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml b/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml new file mode 100644 index 0000000000..6625d4cd62 --- /dev/null +++ b/tests/milvus_benchmark/suites/gpu_stability_sift50m.yaml @@ -0,0 +1,27 @@ +stability: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8h_stability + cache_config.cpu_cache_capacity: 64 + cache_config.cache_insert_data: true + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 100 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_50m_1024_128_l2 + during_time: 480 + search_params: + nprobes: 1-200 + top_ks: 1-200 + nqs: 1-200 + # length of insert vectors + insert_xb: 100000 + # insert after search 4 times + insert_interval: 4 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance.yaml b/tests/milvus_benchmark/suites/insert_performance.yaml new file mode 100644 index 0000000000..f1d487850d --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance.yaml @@ -0,0 +1,19 @@ +insert_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_5m_512_128_l2_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_5m_512_128_l2 + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml b/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml new file mode 100644 index 0000000000..4acbb84099 --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance_deep1b.yaml @@ -0,0 +1,87 @@ +insert_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8 + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_sq8h + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_sq8h + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_pq + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: ivf_pq + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/deep_1b_1024_96_ip_nsg + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: deep_1b_1024_96_ip + ni_per: 100000 + build_index: false + # index_type: nsg + # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml b/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml new file mode 100644 index 0000000000..22b749048c --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance_hamming50m.yaml @@ -0,0 +1,36 @@ +insert_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: hamming_50m_128_512_hamming + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/hamming_50m_128_512_hamming_flat + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: hamming_50m_128_512_hamming + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml b/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml new file mode 100644 index 0000000000..abea53814c --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance_jaccard50m.yaml @@ -0,0 +1,36 @@ +insert_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: jaccard_50m_128_512_jaccard + ni_per: 100000 + build_index: false + # index_type: ivf_flat + # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/jaccard_50m_128_512_jaccard_flat + # cache_config.cpu_cache_capacity: 8 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # table_name: jaccard_50m_128_512_jaccard + # ni_per: 100000 + # build_index: false + # # index_type: ivf_flat + # # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml b/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml new file mode 100644 index 0000000000..e5804a4c89 --- /dev/null +++ b/tests/milvus_benchmark/suites/insert_performance_sift50m.yaml @@ -0,0 +1,87 @@ +insert_performance: + tables: + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_ivf + # cache_config.cpu_cache_capacity: 8 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # table_name: sift_50m_1024_128_ip + # ni_per: 100000 + # build_index: false + # # index_type: ivf_flat + # # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_4096 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + ni_per: 100000 + build_index: false + # index_type: ivf_sq8 + # nlist: 16384 + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_l2_sq8_8192 + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: false + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + gpu_resource_config.build_index_resources: + - gpu0 + table_name: sift_50m_1024_128_l2 + ni_per: 100000 + build_index: false + # index_type: ivf_sq8h + # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_pq + # cache_config.cpu_cache_capacity: 8 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # table_name: sift_50m_1024_128_ip + # ni_per: 100000 + # build_index: false + # # index_type: ivf_pq + # # nlist: 16384 + # - + # server: + # db_config.primary_path: /test/milvus/db_data_gpu/sift_50m_1024_128_ip_nsg + # cache_config.cpu_cache_capacity: 8 + # engine_config.use_blas_threshold: 1100 + # engine_config.gpu_search_threshold: 1 + # gpu_resource_config.enable: false + # gpu_resource_config.cache_capacity: 4 + # gpu_resource_config.search_resources: + # - gpu0 + # gpu_resource_config.build_index_resources: + # - gpu0 + # table_name: sift_50m_1024_128_ip + # ni_per: 100000 + # build_index: false + # # index_type: nsg + # # nlist: 16384 \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/old_performance_sift50m.yaml b/tests/milvus_benchmark/suites/old_performance_sift50m.yaml deleted file mode 100644 index 8e4d35dcac..0000000000 --- a/tests/milvus_benchmark/suites/old_performance_sift50m.yaml +++ /dev/null @@ -1,80 +0,0 @@ -performance: - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["mix_nsg"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_nsg" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_flat"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_ivf" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 3, - # "server.resources": ["gpu0", "gpu1"], - # "server.enable_gpu": True, - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 16, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 200, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.use_gpu_threshold": 100, - # "server.cpu_cache_capacity": 50, - # "server.gpu_cache_capacity": 6, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data_cpu/sift_50m_1024_128_l2_sq8" - # }, - { - "dataset": "sift_50m_1024_128_l2", - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "nprobes": [8, 32], - "top_ks": [1, 16, 64, 128, 256, 512, 1000], - "nqs": [1, 10, 100, 200, 500, 1000], - "server.use_blas_threshold": 1100, - "server.use_gpu_threshold": 100, - "server.cpu_cache_capacity": 50, - "server.gpu_cache_capacity": 6, - "server.resources": ["gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8" - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites/search_debug.yaml b/tests/milvus_benchmark/suites/search_debug.yaml new file mode 100644 index 0000000000..63960827ff --- /dev/null +++ b/tests/milvus_benchmark/suites/search_debug.yaml @@ -0,0 +1,22 @@ +search_performance: + tables: + - + server: + db_config.primary_path: /test/milvus/db_data_gpu/sift_1m_1024_128_l2_ivf + cache_config.cpu_cache_capacity: 8 + engine_config.use_blas_threshold: 1100 + engine_config.gpu_search_threshold: 1 + gpu_resource_config.enable: true + gpu_resource_config.cache_capacity: 4 + gpu_resource_config.search_resources: + - gpu0 + - gpu1 + gpu_resource_config.build_index_resources: + - gpu0 + - gpu1 + table_name: sift_1m_1024_128_l2 + run_count: 3 + search_params: + nprobes: [8, 32] + top_ks: [1, 16] + nqs: [1, 10] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites_accuracy.yaml b/tests/milvus_benchmark/suites_accuracy.yaml deleted file mode 100644 index fd8a5904fa..0000000000 --- a/tests/milvus_benchmark/suites_accuracy.yaml +++ /dev/null @@ -1,121 +0,0 @@ - -accuracy: - # interface: search_vectors - query: - [ - { - "dataset": "random_20m_1024_512_ip", - # index info - "index.index_types": ["flat", "ivf_sq8"], - "index.nlists": [16384], - "index.metric_types": ["ip"], - "nprobes": [1, 16, 64], - "top_ks": [64], - "nqs": [100], - "server.cpu_cache_capacity": 100, - "server.resources": ["cpu", "gpu0"], - "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip", - }, - # { - # "dataset": "sift_50m_1024_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 16, 64], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 160, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2", - # "sift_acc": true - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 16, 64], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 160, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_50m_1024_128_l2_sq8", - # "sift_acc": true - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 16, 64, 128], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h", - # "sift_acc": true - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 16, 64, 128], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h", - # "sift_acc": true - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "index.metric_types": ["l2"], - # "nprobes": [1, 16, 64, 128], - # "top_ks": [64], - # "nqs": [100], - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h", - # "sift_acc": true - # }, - # { - # "dataset": "sift_1m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "sift_10m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 32, - # }, - # { - # "dataset": "sift_50m_1024_128_l2", - # "index.index_types": ["flat", "ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1, 32, 128, 256, 512], - # "nqs": 10, - # "top_ks": 10, - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 64, - # } - - - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites_performance.yaml b/tests/milvus_benchmark/suites_performance.yaml deleted file mode 100644 index 52d457a400..0000000000 --- a/tests/milvus_benchmark/suites_performance.yaml +++ /dev/null @@ -1,258 +0,0 @@ -performance: - - # interface: add_vectors - insert: - # index_type: flat/ivf_flat/ivf_sq8/mix_nsg - [ - # debug - # data_type / data_size / index_file_size / dimension - # data_type: random / ann_sift - # data_size: 10m / 1b - # { - # "table_name": "random_50m_1024_512_ip", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # "server.cpu_cache_capacity": 16, - # # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "table_name": "random_5m_1024_512_ip", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # "server.cpu_cache_capacity": 16, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_5m_1024_512_ip" - # }, - # { - # "table_name": "sift_1m_50_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "table_name": "sift_1m_256_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # "db_path_prefix": "/test/milvus/db_data" - # } - # { - # "table_name": "sift_50m_1024_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # # "server.cpu_cache_capacity": 16, - # }, - # { - # "table_name": "sift_100m_1024_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # }, - # { - # "table_name": "sift_1b_2048_128_l2", - # "ni_per": 100000, - # "processes": 5, # multiprocessing - # "server.cpu_cache_capacity": 16, - # } - ] - - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2_sq8h" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data/sift_1b_2048_128_l2" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 200, - # "server.resources": ["cpu"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - { - "dataset": "random_50m_1024_512_ip", - "index.index_types": ["ivf_sq8h"], - "index.nlists": [16384], - "nprobes": [8], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - "top_ks": [512], - # "nqs": [1, 10, 100, 500, 1000], - "nqs": [500], - "server.use_blas_threshold": 1100, - "server.cpu_cache_capacity": 150, - "server.gpu_cache_capacity": 6, - "server.resources": ["cpu", "gpu0", "gpu1"], - "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip" - }, - # { - # "dataset": "random_50m_1024_512_ip", - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 150, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_50m_1024_512_ip_sq8" - # }, - # { - # "dataset": "random_20m_1024_512_ip", - # "index.index_types": ["flat"], - # "index.nlists": [16384], - # "nprobes": [50], - # "top_ks": [64], - # "nqs": [10], - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 100, - # "server.resources": ["cpu", "gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data/random_20m_1024_512_ip" - # }, - # { - # "dataset": "random_100m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 250, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "dataset": "random_100m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [8, 32], - # "top_ks": [1, 8, 16, 32, 64, 128, 256, 512, 1000], - # "nqs": [1, 10, 100, 500, 1000], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 250, - # "server.resources": ["cpu"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "dataset": "random_10m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "random_10m_1024_512_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 64 - # }, - # { - # "dataset": "sift_500m_1024_128_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 8, 16, 64, 256, 512, 1000], - # "nqs": [1, 100, 500, 800, 1000, 1500], - # # "top_ks": [256], - # # "nqs": [800], - # "processes": 1, # multiprocessing - # # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 120, - # "server.resources": ["gpu0", "gpu1"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8h"], - # "index.nlists": [16384], - # "nprobes": [1], - # # "top_ks": [1], - # # "nqs": [1], - # "top_ks": [256], - # "nqs": [800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 110, - # "server.resources": ["cpu", "gpu0"], - # "db_path_prefix": "/test/milvus/db_data" - # }, - # { - # "dataset": "random_50m_1024_512_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # # "top_ks": [256], - # # "nqs": [800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 128 - # }, - # [ - # { - # "dataset": "sift_1m_50_128_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1], - # "nqs": [1], - # "db_path_prefix": "/test/milvus/db_data" - # # "processes": 1, # multiprocessing - # # "server.use_blas_threshold": 1100, - # # "server.cpu_cache_capacity": 256 - # } - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites_stability.yaml b/tests/milvus_benchmark/suites_stability.yaml deleted file mode 100644 index 408221079e..0000000000 --- a/tests/milvus_benchmark/suites_stability.yaml +++ /dev/null @@ -1,17 +0,0 @@ - -stability: - # interface: search_vectors / add_vectors mix operation - query: - [ - { - "dataset": "random_20m_1024_512_ip", - # "nqs": [1, 10, 100, 1000, 10000], - # "pds": [0.1, 0.44, 0.44, 0.02], - "query_process_num": 10, - # each 10s, do an insertion - # "insert_interval": 1, - # minutes - "during_time": 360, - "server.cpu_cache_capacity": 100 - }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/suites_yzb.yaml b/tests/milvus_benchmark/suites_yzb.yaml deleted file mode 100644 index 59efcb37e4..0000000000 --- a/tests/milvus_benchmark/suites_yzb.yaml +++ /dev/null @@ -1,171 +0,0 @@ -#"server.resources": ["gpu0", "gpu1"] - -performance: - # interface: search_vectors - query: - # dataset: table name you have already created - # key starts with "server." need to reconfig and restart server, including use_blas_threshold/cpu_cache_capacity .. - [ - # debug - # { - # "dataset": "random_10m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "random_10m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "random_10m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "random_10m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 16, - # }, - # { - # "dataset": "random_10m_1024_512_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 64 - # }, -# { -# "dataset": "sift_50m_1024_128_l2", -# # index info -# "index.index_types": ["ivf_sq8"], -# "index.nlists": [16384], -# "nprobes": [1, 32, 128], -# "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], -# "nqs": [1, 10, 100, 500, 800], -# # "top_ks": [256], -# # "nqs": [800], -# "processes": 1, # multiprocessing -# "server.use_blas_threshold": 1100, -# "server.cpu_cache_capacity": 310, -# "server.resources": ["gpu0", "gpu1"] -# }, - { - "dataset": "sift_1m_1024_128_l2", - # index info - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "nprobes": [32], - "top_ks": [10], - "nqs": [100], - # "top_ks": [256], - # "nqs": [800], - "processes": 1, # multiprocessing - "server.use_blas_threshold": 1100, - "server.cpu_cache_capacity": 310, - "server.resources": ["cpu"] - }, - { - "dataset": "sift_1m_1024_128_l2", - # index info - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "nprobes": [32], - "top_ks": [10], - "nqs": [100], - # "top_ks": [256], - # "nqs": [800], - "processes": 1, # multiprocessing - "server.use_blas_threshold": 1100, - "server.cpu_cache_capacity": 310, - "server.resources": ["gpu0"] - }, - { - "dataset": "sift_1m_1024_128_l2", - # index info - "index.index_types": ["ivf_sq8"], - "index.nlists": [16384], - "nprobes": [32], - "top_ks": [10], - "nqs": [100], - # "top_ks": [256], - # "nqs": [800], - "processes": 1, # multiprocessing - "server.use_blas_threshold": 1100, - "server.cpu_cache_capacity": 310, - "server.resources": ["gpu0", "gpu1"] - }, - # { - # "dataset": "sift_1b_2048_128_l2", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # # "top_ks": [256], - # # "nqs": [800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 310 - # }, -# { -# "dataset": "random_50m_1024_512_l2", -# # index info -# "index.index_types": ["ivf_sq8"], -# "index.nlists": [16384], -# "nprobes": [1], -# "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], -# "nqs": [1, 10, 100, 500, 800], -# # "top_ks": [256], -# # "nqs": [800], -# "processes": 1, # multiprocessing -# "server.use_blas_threshold": 1100, -# "server.cpu_cache_capacity": 128, -# "server.resources": ["gpu0", "gpu1"] -# }, - # { - # "dataset": "random_100m_1024_512_ip", - # # index info - # "index.index_types": ["ivf_sq8"], - # "index.nlists": [16384], - # "nprobes": [1], - # "top_ks": [1, 2, 4, 8, 16, 32, 64, 128, 256], - # "nqs": [1, 10, 100, 500, 800], - # "processes": 1, # multiprocessing - # "server.use_blas_threshold": 1100, - # "server.cpu_cache_capacity": 256 - # }, - ] \ No newline at end of file diff --git a/tests/milvus_benchmark/utils.py b/tests/milvus_benchmark/utils.py index ddcd215237..76bf04a873 100644 --- a/tests/milvus_benchmark/utils.py +++ b/tests/milvus_benchmark/utils.py @@ -7,22 +7,34 @@ import os import sys import pdb import time +import json import datetime import argparse import threading import logging -import docker -import multiprocessing -import numpy +import string +import random +# import multiprocessing +# import numpy # import psutil -from yaml import load, dump +import h5py +# import docker +from yaml import full_load, dump import tableprint as tp +from pprint import pprint +from kubernetes import client, config + logger = logging.getLogger("milvus_benchmark.utils") +config.load_kube_config() MULTI_DB_SLAVE_PATH = "/opt/milvus/data2;/opt/milvus/data3" +def get_unique_name(): + return "benchmark-test-"+"".join(random.choice(string.ascii_letters + string.digits) for _ in range(8)).lower() + + def get_current_time(): return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) @@ -36,11 +48,18 @@ def print_table(headers, columns, data): tp.table(bodys, headers) +def get_dataset(hdf5_file_path): + if not os.path.exists(hdf5_file_path): + raise Exception("%s not existed" % hdf5_file_path) + dataset = h5py.File(hdf5_file_path) + return dataset + + def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave=None): if not os.path.isfile(file_path): raise Exception('File: %s not found' % file_path) with open(file_path) as f: - config_dict = load(f) + config_dict = full_load(f) f.close() if config_dict: if k.find("use_blas_threshold") != -1: @@ -67,132 +86,258 @@ def modify_config(k, v, type=None, file_path="conf/server_config.yaml", db_slave raise Exception('Load file:%s error' % file_path) -def pull_image(image): - registry = image.split(":")[0] - image_tag = image.split(":")[1] - client = docker.APIClient(base_url='unix://var/run/docker.sock') - logger.info("Start pulling image: %s" % image) - return client.pull(registry, image_tag) +# update server_config.yaml +def update_server_config(file_path, server_config): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + with open(file_path) as f: + values_dict = full_load(f) + f.close() + for k, v in server_config.items(): + if k.find("primary_path") != -1: + values_dict["db_config"]["primary_path"] = v + elif k.find("use_blas_threshold") != -1: + values_dict['engine_config']['use_blas_threshold'] = int(v) + elif k.find("gpu_search_threshold") != -1: + values_dict['engine_config']['gpu_search_threshold'] = int(v) + elif k.find("cpu_cache_capacity") != -1: + values_dict['cache_config']['cpu_cache_capacity'] = int(v) + elif k.find("cache_insert_data") != -1: + values_dict['cache_config']['cache_insert_data'] = v + elif k.find("enable") != -1: + values_dict['gpu_resource_config']['enable'] = v + elif k.find("gpu_cache_capacity") != -1: + values_dict['gpu_resource_config']['cache_capacity'] = int(v) + elif k.find("build_index_resources") != -1: + values_dict['gpu_resource_config']['build_index_resources'] = v + elif k.find("search_resources") != -1: + values_dict['gpu_resource_config']['search_resources'] = v + with open(file_path, 'w') as f: + dump(values_dict, f, default_flow_style=False) + f.close() -def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None): - import colors - - client = docker.from_env() - # if mem_limit is None: - # mem_limit = psutil.virtual_memory().available - # logger.info('Memory limit:', mem_limit) - # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1) - # logger.info('Running on CPUs:', cpu_limit) - for dir_item in ['logs', 'db']: - try: - os.mkdir(os.path.abspath(dir_item)) - except Exception as e: - pass - - if test_type == "local": - volumes = { - os.path.abspath('conf'): - {'bind': '/opt/milvus/conf', 'mode': 'ro'}, - os.path.abspath('logs'): - {'bind': '/opt/milvus/logs', 'mode': 'rw'}, - os.path.abspath('db'): - {'bind': '/opt/milvus/db', 'mode': 'rw'}, +# update values.yaml +def update_values(file_path, hostname): + if not os.path.isfile(file_path): + raise Exception('File: %s not found' % file_path) + with open(file_path) as f: + values_dict = full_load(f) + f.close() + if values_dict['engine']['nodeSelector']: + logger.warning("nodeSelector has been set: %s" % str(values_dict['engine']['nodeSelector'])) + return + # update values.yaml with the given host + # set limit/request cpus in resources + v1 = client.CoreV1Api() + node = v1.read_node(hostname) + cpus = node.status.allocatable.get("cpu") + # DEBUG + values_dict['engine']['resources'] = { + "limits": { + "cpu": str(int(cpus)-1)+".0" + }, + "requests": { + "cpu": str(int(cpus)-2)+".0" } - elif test_type == "remote": - if volume_name is None: - raise Exception("No volume name") - remote_log_dir = volume_name+'/logs' - remote_db_dir = volume_name+'/db' - - for dir_item in [remote_log_dir, remote_db_dir]: - if not os.path.isdir(dir_item): - os.makedirs(dir_item, exist_ok=True) - volumes = { - os.path.abspath('conf'): - {'bind': '/opt/milvus/conf', 'mode': 'ro'}, - remote_log_dir: - {'bind': '/opt/milvus/logs', 'mode': 'rw'}, - remote_db_dir: - {'bind': '/opt/milvus/db', 'mode': 'rw'} + } + values_dict['engine']['nodeSelector'] = {'kubernetes.io/hostname': hostname} + values_dict['engine']['tolerations'].append({ + "key": "worker", + "operator": "Equal", + "value": "performance", + "effect": "NoSchedule" + }) + # add extra volumes + values_dict['extraVolumes'].append({ + 'name': 'test', + 'flexVolume': { + 'driver': "fstab/cifs", + 'fsType': "cifs", + 'secretRef': { + 'name': "cifs-test-secret" + }, + 'options': { + 'networkPath': "//192.168.1.126/test", + 'mountOptions': "vers=1.0" + } } - # add volumes - if db_slave and isinstance(db_slave, int): - for i in range(2, db_slave+1): - remote_db_dir = volume_name+'/data'+str(i) - if not os.path.isdir(remote_db_dir): - os.makedirs(remote_db_dir, exist_ok=True) - volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'} - - container = client.containers.run( - image, - volumes=volumes, - runtime="nvidia", - ports={'19530/tcp': 19530, '8080/tcp': 8080}, - # environment=["OMP_NUM_THREADS=48"], - # cpuset_cpus=cpu_limit, - # mem_limit=mem_limit, - # environment=[""], - detach=True) - - def stream_logs(): - for line in container.logs(stream=True): - logger.info(colors.color(line.decode().rstrip(), fg='blue')) - - if sys.version_info >= (3, 0): - t = threading.Thread(target=stream_logs, daemon=True) - else: - t = threading.Thread(target=stream_logs) - t.daemon = True - t.start() - - logger.info('Container: %s started' % container) - return container - # exit_code = container.wait(timeout=timeout) - # # Exit if exit code - # if exit_code == 0: - # return container - # elif exit_code is not None: - # print(colors.color(container.logs().decode(), fg='red')) - # raise Exception('Child process raised exception %s' % str(exit_code)) - -def restart_server(container): - client = docker.APIClient(base_url='unix://var/run/docker.sock') - - client.restart(container.name) - logger.info('Container: %s restarted' % container.name) - return container + }) + values_dict['extraVolumeMounts'].append({ + 'name': 'test', + 'mountPath': '/test' + }) + with open(file_path, 'w') as f: + dump(values_dict, f, default_flow_style=False) + f.close() -def remove_container(container): - container.remove(force=True) - logger.info('Container: %s removed' % container) +# deploy server +def helm_install_server(helm_path, image_tag, image_type, name, namespace): + timeout = 180 + install_cmd = "helm install --wait --timeout %d \ + --set engine.image.tag=%s \ + --set expose.type=clusterIP \ + --name %s \ + -f ci/db_backend/sqlite_%s_values.yaml \ + -f ci/filebeat/values.yaml \ + --namespace %s \ + --version 0.0 ." % (timeout, image_tag, name, image_type, namespace) + logger.debug(install_cmd) + if os.system("cd %s && %s" % (helm_path, install_cmd)): + logger.error("Helm install failed") + return None + time.sleep(5) + v1 = client.CoreV1Api() + host = "%s-milvus-engine.%s.svc.cluster.local" % (name, namespace) + pod_name = None + pod_id = None + pods = v1.list_namespaced_pod(namespace) + for i in pods.items: + if i.metadata.name.find(name) != -1: + pod_name = i.metadata.name + pod_ip = i.status.pod_ip + logger.debug(pod_name) + logger.debug(pod_ip) + return pod_name, pod_ip -def remove_all_containers(image): - client = docker.from_env() - try: - for container in client.containers.list(): - if image in container.image.tags: - container.stop(timeout=30) - container.remove(force=True) - except Exception as e: - logger.error("Containers removed failed") +# delete server +def helm_del_server(name): + del_cmd = "helm del --purge %s" % name + logger.debug(del_cmd) + if os.system(del_cmd): + logger.error("Helm delete name:%s failed" % name) + return False + return True -def container_exists(image): - ''' - Check if container existed with the given image name - @params: image name - @return: container if exists - ''' - res = False - client = docker.from_env() - for container in client.containers.list(): - if image in container.image.tags: - # True - res = container - return res +# def pull_image(image): +# registry = image.split(":")[0] +# image_tag = image.split(":")[1] +# client = docker.APIClient(base_url='unix://var/run/docker.sock') +# logger.info("Start pulling image: %s" % image) +# return client.pull(registry, image_tag) + + +# def run_server(image, mem_limit=None, timeout=30, test_type="local", volume_name=None, db_slave=None): +# import colors + +# client = docker.from_env() +# # if mem_limit is None: +# # mem_limit = psutil.virtual_memory().available +# # logger.info('Memory limit:', mem_limit) +# # cpu_limit = "0-%d" % (multiprocessing.cpu_count() - 1) +# # logger.info('Running on CPUs:', cpu_limit) +# for dir_item in ['logs', 'db']: +# try: +# os.mkdir(os.path.abspath(dir_item)) +# except Exception as e: +# pass + +# if test_type == "local": +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# os.path.abspath('logs'): +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# os.path.abspath('db'): +# {'bind': '/opt/milvus/db', 'mode': 'rw'}, +# } +# elif test_type == "remote": +# if volume_name is None: +# raise Exception("No volume name") +# remote_log_dir = volume_name+'/logs' +# remote_db_dir = volume_name+'/db' + +# for dir_item in [remote_log_dir, remote_db_dir]: +# if not os.path.isdir(dir_item): +# os.makedirs(dir_item, exist_ok=True) +# volumes = { +# os.path.abspath('conf'): +# {'bind': '/opt/milvus/conf', 'mode': 'ro'}, +# remote_log_dir: +# {'bind': '/opt/milvus/logs', 'mode': 'rw'}, +# remote_db_dir: +# {'bind': '/opt/milvus/db', 'mode': 'rw'} +# } +# # add volumes +# if db_slave and isinstance(db_slave, int): +# for i in range(2, db_slave+1): +# remote_db_dir = volume_name+'/data'+str(i) +# if not os.path.isdir(remote_db_dir): +# os.makedirs(remote_db_dir, exist_ok=True) +# volumes[remote_db_dir] = {'bind': '/opt/milvus/data'+str(i), 'mode': 'rw'} + +# container = client.containers.run( +# image, +# volumes=volumes, +# runtime="nvidia", +# ports={'19530/tcp': 19530, '8080/tcp': 8080}, +# # environment=["OMP_NUM_THREADS=48"], +# # cpuset_cpus=cpu_limit, +# # mem_limit=mem_limit, +# # environment=[""], +# detach=True) + +# def stream_logs(): +# for line in container.logs(stream=True): +# logger.info(colors.color(line.decode().rstrip(), fg='blue')) + +# if sys.version_info >= (3, 0): +# t = threading.Thread(target=stream_logs, daemon=True) +# else: +# t = threading.Thread(target=stream_logs) +# t.daemon = True +# t.start() + +# logger.info('Container: %s started' % container) +# return container +# # exit_code = container.wait(timeout=timeout) +# # # Exit if exit code +# # if exit_code == 0: +# # return container +# # elif exit_code is not None: +# # print(colors.color(container.logs().decode(), fg='red')) +# # raise Exception('Child process raised exception %s' % str(exit_code)) + +# def restart_server(container): +# client = docker.APIClient(base_url='unix://var/run/docker.sock') + +# client.restart(container.name) +# logger.info('Container: %s restarted' % container.name) +# return container + + +# def remove_container(container): +# container.remove(force=True) +# logger.info('Container: %s removed' % container) + + +# def remove_all_containers(image): +# client = docker.from_env() +# try: +# for container in client.containers.list(): +# if image in container.image.tags: +# container.stop(timeout=30) +# container.remove(force=True) +# except Exception as e: +# logger.error("Containers removed failed") + + +# def container_exists(image): +# ''' +# Check if container existed with the given image name +# @params: image name +# @return: container if exists +# ''' +# res = False +# client = docker.from_env() +# for container in client.containers.list(): +# if image in container.image.tags: +# # True +# res = container +# return res if __name__ == '__main__': From 4c2276a91c5c0aac01060d95b702ec2dc3ef8ee1 Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Tue, 7 Jan 2020 17:04:15 +0800 Subject: [PATCH 06/14] Update CONTRIBUTING.md (#942) * [skip ci]#668 - Update badge of README * Fix README.md * Fix README.md * Merge remote-tracking branch 'upstream/master' * #910 Change c++ standard to c++17 * Remove unused headers * #910 Change Milvus c++ standard to c++17 * #910 Change Milvus c++ standard to c++17 * #941 Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1ff37372db..24354c5d84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,7 @@ Contributions to Milvus fall into the following categories. ### Contributing code -If you have improvements to Milvus, send us your pull requests! For those just getting started, see [GitHub workflow](#github-workflow). +If you have improvements to Milvus, send us your pull requests! For those just getting started, see [GitHub workflow](#github-workflow). Make sure to refer to the related issue in your pull request's comment and update CHANGELOG.md. The Milvus team members will review your pull requests, and once it is accepted, the status of the projects to which it is associated will be changed to **Reviewer approved**. This means we are working on submitting your pull request to the internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub. From a1d722a64a6232e1e6ab08389fb284d587ab8022 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Tue, 7 Jan 2020 17:57:27 +0800 Subject: [PATCH 07/14] Create pull request template (#944) * Update README.md * [skip ci] Add IVFLAT links * [skip ci] Minor * Create PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000000..e4cde371cf --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,21 @@ +**What type of PR is this?** + +api-change / bug / design / documentation / feature + + +**What this PR does / why we need it:** + + +**Which issue(s) this PR fixes:** + +Fixes # + + +**Special notes for your reviewer:** + + +**Does this PR introduce a user-facing change?:** + + +**Additional documentation (e.g. design docs, usage docs, etc.):** + From 988dc399ace2dbc3bd87a99259909ff6f991caae Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Tue, 7 Jan 2020 18:29:16 +0800 Subject: [PATCH 08/14] Add CODEOWNERS file (#945) * [skip ci]#668 - Update badge of README * Fix README.md * Fix README.md * Merge remote-tracking branch 'upstream/master' * #910 Change c++ standard to c++17 * Remove unused headers * #910 Change Milvus c++ standard to c++17 * #910 Change Milvus c++ standard to c++17 * #941 Update CONTRIBUTING.md * [skip ci] Add CODEOWNERS file #943 * Add bestpractice badge --- CODEOWNERS | 10 ++++++++++ README.md | 1 + 2 files changed, 11 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000000..53707e7d0e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,10 @@ +# Each line is a component followed by one or more owners. + +* @JinHai-CN +/core @JinHai-CN @XuPeng-SH +/ci @ZhifengZhang-CN +/docker @ZhifengZhang-CN +/docs @jielinxu @yamasite +/sdk @fishpenguin +/shards @XuPeng-SH +/tests @del-zhenwu \ No newline at end of file diff --git a/README.md b/README.md index 254a582036..9a4741d8b0 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ ![Release_date](https://img.shields.io/badge/release%20date-December-yellowgreen) [![codecov](https://codecov.io/gh/milvus-io/milvus/branch/master/graph/badge.svg)](https://codecov.io/gh/milvus-io/milvus) [![All Contributors](https://img.shields.io/badge/all_contributors-6-orange.svg?style=flat-square)](#contributors-) +[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3563/badge)](https://bestpractices.coreinfrastructure.org/projects/3563) [中文版](README_CN.md) | [日本語版](README_JP.md) From 3c02b4b58130a9098b67136c005d48156119cd42 Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Wed, 8 Jan 2020 10:39:05 +0800 Subject: [PATCH 09/14] add s3 mock for unittest (#937) * #815 change S3 client_ptr_ to shared_ptr * #815 install test_storage to unittest * #815 add MockS3Client.h for unittest * #815 optimize MockS3Client.h * #815 update unittest * #815 update unittest * #815 fix clang-format * #815 use FIU for unittest * #815 enable FIU in jenkins * #815 update unittest * #815 enable FIU in docker build --- ci/jenkins/step/build.groovy | 4 +- ci/scripts/build.sh | 10 +- core/src/db/Utils.cpp | 2 +- core/src/storage/s3/S3ClientMock.h | 111 +++++++++++++++++ core/src/storage/s3/S3ClientWrapper.cpp | 30 +++-- core/src/storage/s3/S3ClientWrapper.h | 8 +- core/unittest/storage/CMakeLists.txt | 2 +- core/unittest/storage/test_s3_client.cpp | 145 +++++++++++++++++++++-- core/unittest/storage/utils.cpp | 7 +- docker-compose.yml | 4 +- 10 files changed, 289 insertions(+), 34 deletions(-) create mode 100644 core/src/storage/s3/S3ClientMock.h diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index b3a8a49278..aaa106376f 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -4,9 +4,9 @@ timeout(time: 75, unit: 'MINUTES') { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${BINARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c -i\"" } else { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c -i\"" } sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index f30644a3f2..0ac615891e 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -21,9 +21,10 @@ BUILD_COVERAGE="OFF" RUN_CPPLINT="OFF" GPU_VERSION="OFF" WITH_MKL="OFF" +FIU_ENABLE="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:pgulcmh" arg +while getopts "o:t:b:f:pgulcmih" arg do case $arg in o) @@ -57,6 +58,9 @@ do m) WITH_MKL="ON" ;; + i) + FIU_ENABLE="ON" + ;; h) # help echo " @@ -71,10 +75,11 @@ parameter: -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) -m: build with MKL(default: OFF) +-i: build FIU_ENABLE(default: OFF) -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-m] [-i] [-h] " exit 0 ;; @@ -105,6 +110,7 @@ CMAKE_CMD="cmake \ -DFAISS_WITH_MKL=${WITH_MKL} \ -DArrow_SOURCE=AUTO \ -DFAISS_SOURCE=AUTO \ +-DMILVUS_WITH_FIU=${FIU_ENABLE} \ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} diff --git a/core/src/db/Utils.cpp b/core/src/db/Utils.cpp index 62c71caa7c..0bcfba7025 100644 --- a/core/src/db/Utils.cpp +++ b/core/src/db/Utils.cpp @@ -126,7 +126,7 @@ DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool if (minio_enable) { std::string table_path = options.path_ + TABLES_FOLDER + table_id; - auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); Status stat = storage_inst.DeleteObjects(table_path); if (!stat.ok()) { return stat; diff --git a/core/src/storage/s3/S3ClientMock.h b/core/src/storage/s3/S3ClientMock.h new file mode 100644 index 0000000000..d7509e0490 --- /dev/null +++ b/core/src/storage/s3/S3ClientMock.h @@ -0,0 +1,111 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace milvus { +namespace storage { + +/* + * This is a class that represents a S3 Client which is used to mimic the put/get operations of a actual s3 client. + * During a put object, the body of the request is stored as well as the metadata of the request. This data is then + * populated into a get object result when a get operation is called. + */ +class S3ClientMock : public Aws::S3::S3Client { + public: + explicit S3ClientMock(Aws::Client::ClientConfiguration clientConfiguration = Aws::Client::ClientConfiguration()) + : S3Client(Aws::Auth::AWSCredentials("", ""), clientConfiguration) { + } + + Aws::S3::Model::CreateBucketOutcome + CreateBucket(const Aws::S3::Model::CreateBucketRequest& request) const override { + Aws::S3::Model::CreateBucketResult result; + return Aws::S3::Model::CreateBucketOutcome(std::move(result)); + } + + Aws::S3::Model::DeleteBucketOutcome + DeleteBucket(const Aws::S3::Model::DeleteBucketRequest& request) const override { + Aws::NoResult result; + return Aws::S3::Model::DeleteBucketOutcome(std::move(result)); + } + + Aws::S3::Model::PutObjectOutcome + PutObject(const Aws::S3::Model::PutObjectRequest& request) const override { + Aws::String key = request.GetKey(); + std::shared_ptr body = request.GetBody(); + aws_map_[key] = body; + + Aws::S3::Model::PutObjectResult result; + return Aws::S3::Model::PutObjectOutcome(std::move(result)); + } + + Aws::S3::Model::GetObjectOutcome + GetObject(const Aws::S3::Model::GetObjectRequest& request) const override { + auto factory = request.GetResponseStreamFactory(); + Aws::Utils::Stream::ResponseStream resp_stream(factory); + + try { + std::shared_ptr body = aws_map_.at(request.GetKey()); + Aws::String body_str((Aws::IStreamBufIterator(*body)), Aws::IStreamBufIterator()); + + resp_stream.GetUnderlyingStream().write(body_str.c_str(), body_str.length()); + resp_stream.GetUnderlyingStream().flush(); + Aws::AmazonWebServiceResult awsStream( + std::move(resp_stream), Aws::Http::HeaderValueCollection()); + + Aws::S3::Model::GetObjectResult result(std::move(awsStream)); + return Aws::S3::Model::GetObjectOutcome(std::move(result)); + } catch (...) { + return Aws::S3::Model::GetObjectOutcome(); + } + } + + Aws::S3::Model::ListObjectsOutcome + ListObjects(const Aws::S3::Model::ListObjectsRequest& request) const override { + /* TODO: add object key list into ListObjectsOutcome */ + + Aws::S3::Model::ListObjectsResult result; + return Aws::S3::Model::ListObjectsOutcome(std::move(result)); + } + + Aws::S3::Model::DeleteObjectOutcome + DeleteObject(const Aws::S3::Model::DeleteObjectRequest& request) const override { + Aws::String key = request.GetKey(); + aws_map_.erase(key); + Aws::S3::Model::DeleteObjectResult result; + Aws::S3::Model::DeleteObjectOutcome(std::move(result)); + return result; + } + + mutable Aws::Map> aws_map_; +}; + +} // namespace storage +} // namespace milvus diff --git a/core/src/storage/s3/S3ClientWrapper.cpp b/core/src/storage/s3/S3ClientWrapper.cpp index 620b97ae95..e7d9251b9f 100644 --- a/core/src/storage/s3/S3ClientWrapper.cpp +++ b/core/src/storage/s3/S3ClientWrapper.cpp @@ -22,12 +22,14 @@ #include #include #include +#include #include #include #include #include #include "server/Config.h" +#include "storage/s3/S3ClientMock.h" #include "storage/s3/S3ClientWrapper.h" #include "utils/Error.h" #include "utils/Log.h" @@ -40,6 +42,7 @@ S3ClientWrapper::StartService() { server::Config& config = server::Config::GetInstance(); bool minio_enable = false; CONFIG_CHECK(config.GetStorageConfigMinioEnable(minio_enable)); + fiu_do_on("S3ClientWrapper.StartService.minio_disable", minio_enable = false); if (!minio_enable) { STORAGE_LOG_INFO << "MinIO not enabled!"; return Status::OK(); @@ -52,29 +55,30 @@ S3ClientWrapper::StartService() { CONFIG_CHECK(config.GetStorageConfigMinioBucket(minio_bucket_)); Aws::InitAPI(options_); - Aws::Client::ClientConfiguration cfg; + Aws::Client::ClientConfiguration cfg; cfg.endpointOverride = minio_address_ + ":" + minio_port_; cfg.scheme = Aws::Http::Scheme::HTTP; cfg.verifySSL = false; - client_ptr_ = new Aws::S3::S3Client(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); - if (client_ptr_ == nullptr) { - std::string str = "Cannot connect S3 server."; - return milvus::Status(SERVER_UNEXPECTED_ERROR, str); + client_ptr_ = + std::make_shared(Aws::Auth::AWSCredentials(minio_access_key_, minio_secret_key_), cfg, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false); + + bool mock_enable = false; + fiu_do_on("S3ClientWrapper.StartService.mock_enable", mock_enable = true); + if (mock_enable) { + client_ptr_ = std::make_shared(); } return CreateBucket(); } -Status +void S3ClientWrapper::StopService() { if (client_ptr_ != nullptr) { - delete client_ptr_; client_ptr_ = nullptr; } Aws::ShutdownAPI(options_); - return Status::OK(); } Status @@ -84,6 +88,7 @@ S3ClientWrapper::CreateBucket() { auto outcome = client_ptr_->CreateBucket(request); + fiu_do_on("S3ClientWrapper.CreateBucket.outcome.fail", outcome = Aws::S3::Model::CreateBucketOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); if (err.GetErrorType() != Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU) { @@ -103,6 +108,7 @@ S3ClientWrapper::DeleteBucket() { auto outcome = client_ptr_->DeleteBucket(request); + fiu_do_on("S3ClientWrapper.DeleteBucket.outcome.fail", outcome = Aws::S3::Model::DeleteBucketOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: DeleteBucket: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -131,6 +137,7 @@ S3ClientWrapper::PutObjectFile(const std::string& object_name, const std::string auto outcome = client_ptr_->PutObject(request); + fiu_do_on("S3ClientWrapper.PutObjectFile.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -152,6 +159,7 @@ S3ClientWrapper::PutObjectStr(const std::string& object_name, const std::string& auto outcome = client_ptr_->PutObject(request); + fiu_do_on("S3ClientWrapper.PutObjectStr.outcome.fail", outcome = Aws::S3::Model::PutObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: PutObject: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -169,6 +177,7 @@ S3ClientWrapper::GetObjectFile(const std::string& object_name, const std::string auto outcome = client_ptr_->GetObject(request); + fiu_do_on("S3ClientWrapper.GetObjectFile.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -191,6 +200,7 @@ S3ClientWrapper::GetObjectStr(const std::string& object_name, std::string& conte auto outcome = client_ptr_->GetObject(request); + fiu_do_on("S3ClientWrapper.GetObjectStr.outcome.fail", outcome = Aws::S3::Model::GetObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -217,6 +227,7 @@ S3ClientWrapper::ListObjects(std::vector& object_list, const std::s auto outcome = client_ptr_->ListObjects(request); + fiu_do_on("S3ClientWrapper.ListObjects.outcome.fail", outcome = Aws::S3::Model::ListObjectsOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: ListObjects: " << err.GetExceptionName() << ": " << err.GetMessage(); @@ -244,6 +255,7 @@ S3ClientWrapper::DeleteObject(const std::string& object_name) { auto outcome = client_ptr_->DeleteObject(request); + fiu_do_on("S3ClientWrapper.DeleteObject.outcome.fail", outcome = Aws::S3::Model::DeleteObjectOutcome()); if (!outcome.IsSuccess()) { auto err = outcome.GetError(); STORAGE_LOG_ERROR << "ERROR: DeleteObject: " << err.GetExceptionName() << ": " << err.GetMessage(); diff --git a/core/src/storage/s3/S3ClientWrapper.h b/core/src/storage/s3/S3ClientWrapper.h index 6f2dc1e0be..0e1a99e77d 100644 --- a/core/src/storage/s3/S3ClientWrapper.h +++ b/core/src/storage/s3/S3ClientWrapper.h @@ -19,6 +19,7 @@ #include #include +#include #include #include #include "storage/IStorage.h" @@ -28,9 +29,6 @@ namespace storage { class S3ClientWrapper : public IStorage { public: - S3ClientWrapper() = default; - ~S3ClientWrapper() = default; - static S3ClientWrapper& GetInstance() { static S3ClientWrapper wrapper; @@ -39,7 +37,7 @@ class S3ClientWrapper : public IStorage { Status StartService(); - Status + void StopService(); Status @@ -62,7 +60,7 @@ class S3ClientWrapper : public IStorage { DeleteObjects(const std::string& marker) override; private: - Aws::S3::S3Client* client_ptr_ = nullptr; + std::shared_ptr client_ptr_; Aws::SDKOptions options_; std::string minio_address_; diff --git a/core/unittest/storage/CMakeLists.txt b/core/unittest/storage/CMakeLists.txt index 10071facb4..dec0e35098 100644 --- a/core/unittest/storage/CMakeLists.txt +++ b/core/unittest/storage/CMakeLists.txt @@ -41,4 +41,4 @@ target_link_libraries(test_storage ${unittest_libs} ) -install(TARGETS test_storage DESTINATION bin) \ No newline at end of file +install(TARGETS test_storage DESTINATION unittest) \ No newline at end of file diff --git a/core/unittest/storage/test_s3_client.cpp b/core/unittest/storage/test_s3_client.cpp index 0dda5fd7e6..c4899eaddd 100644 --- a/core/unittest/storage/test_s3_client.cpp +++ b/core/unittest/storage/test_s3_client.cpp @@ -19,27 +19,31 @@ #include #include #include +#include +#include #include "easyloggingpp/easylogging++.h" #include "server/Config.h" -#include "storage/IStorage.h" #include "storage/s3/S3ClientWrapper.h" +#include "storage/s3/S3IOReader.h" +#include "storage/s3/S3IOWriter.h" +#include "storage/IStorage.h" #include "storage/utils.h" INITIALIZE_EASYLOGGINGPP TEST_F(StorageTest, S3_CLIENT_TEST) { + fiu_init(0); + const std::string filename = "/tmp/test_file_in"; + const std::string filename_dummy = "/tmp/test_file_dummy"; const std::string filename_out = "/tmp/test_file_out"; - const std::string object_name = "/tmp/test_obj"; + const std::string objname = "/tmp/test_obj"; + const std::string objname_dummy = "/tmp/test_obj_dummy"; const std::string content = "abcdefghijklmnopqrstuvwxyz"; - std::string config_path(CONFIG_PATH); - config_path += CONFIG_FILE; - milvus::server::Config& config = milvus::server::Config::GetInstance(); - ASSERT_TRUE(config.LoadConfigFile(config_path).ok()); - - auto storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0); ASSERT_TRUE(storage_inst.StartService().ok()); /////////////////////////////////////////////////////////////////////////// @@ -64,18 +68,137 @@ TEST_F(StorageTest, S3_CLIENT_TEST) { /////////////////////////////////////////////////////////////////////////// /* check PutObjectStr() and GetObjectStr() */ { - ASSERT_TRUE(storage_inst.PutObjectStr(object_name, content).ok()); + ASSERT_TRUE(storage_inst.PutObjectStr(objname, content).ok()); std::string content_out; - ASSERT_TRUE(storage_inst.GetObjectStr(object_name, content_out).ok()); + ASSERT_TRUE(storage_inst.GetObjectStr(objname, content_out).ok()); ASSERT_TRUE(content_out == content); } /////////////////////////////////////////////////////////////////////////// + ASSERT_TRUE(storage_inst.DeleteObject(filename).ok()); + ASSERT_TRUE(storage_inst.DeleteObject(objname).ok()); + ASSERT_TRUE(storage_inst.DeleteObjects("/tmp").ok()); ASSERT_TRUE(storage_inst.DeleteBucket().ok()); - ASSERT_TRUE(storage_inst.StopService().ok()); + storage_inst.StopService(); } +TEST_F(StorageTest, S3_RW_TEST) { + fiu_init(0); + + const std::string index_name = "/tmp/test_index"; + const std::string content = "abcdefg"; + + auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0); + ASSERT_TRUE(storage_inst.StartService().ok()); + + { + milvus::storage::S3IOWriter writer(index_name); + size_t len = content.length(); + writer.write(&len, sizeof(len)); + writer.write((void*)(content.data()), len); + ASSERT_TRUE(len + sizeof(len) == writer.length()); + } + + { + milvus::storage::S3IOReader reader(index_name); + size_t length = reader.length(); + size_t rp = 0; + reader.seekg(rp); + std::string content_out; + while (rp < length) { + size_t len; + reader.read(&len, sizeof(len)); + rp += sizeof(len); + reader.seekg(rp); + + auto data = new char[len]; + reader.read(data, len); + rp += len; + reader.seekg(rp); + + content_out += std::string(data, len); + + delete[] data; + } + + ASSERT_TRUE(content == content_out); + } + + storage_inst.StopService(); +} + +TEST_F(StorageTest, S3_FAIL_TEST) { + fiu_init(0); + + const std::string filename = "/tmp/test_file_in"; + const std::string filename_dummy = "/tmp/test_file_dummy"; + const std::string filename_out = "/tmp/test_file_out"; + const std::string objname = "/tmp/test_obj"; + const std::string objname_dummy = "/tmp/test_obj_dummy"; + const std::string content = "abcdefghijklmnopqrstuvwxyz"; + + auto& storage_inst = milvus::storage::S3ClientWrapper::GetInstance(); + + fiu_enable("S3ClientWrapper.StartService.minio_disable", 1, NULL, 0); + ASSERT_TRUE(storage_inst.StartService().ok()); + fiu_disable("S3ClientWrapper.StartService.minio_disable"); + + fiu_enable("S3ClientWrapper.StartService.mock_enable", 1, NULL, 0); + ASSERT_TRUE(storage_inst.StartService().ok()); + fiu_disable("S3ClientWrapper.StartService.mock_enable"); + + fiu_enable("S3ClientWrapper.CreateBucket.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.CreateBucket().ok()); + fiu_disable("S3ClientWrapper.CreateBucket.outcome.fail"); + + /////////////////////////////////////////////////////////////////////////// + /* check PutObjectFile() and GetObjectFile() */ + { + fiu_enable("S3ClientWrapper.PutObjectFile.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.PutObjectFile(filename, filename).ok()); + fiu_disable("S3ClientWrapper.PutObjectFile.outcome.fail"); + + fiu_enable("S3ClientWrapper.GetObjectFile.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.GetObjectFile(filename, filename_out).ok()); + fiu_disable("S3ClientWrapper.GetObjectFile.outcome.fail"); + + ASSERT_FALSE(storage_inst.PutObjectFile(filename_dummy, filename_dummy).ok()); + ASSERT_FALSE(storage_inst.GetObjectFile(filename_dummy, filename_out).ok()); + } + + /////////////////////////////////////////////////////////////////////////// + /* check PutObjectStr() and GetObjectStr() */ + { + fiu_enable("S3ClientWrapper.PutObjectStr.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.PutObjectStr(objname, content).ok()); + fiu_disable("S3ClientWrapper.PutObjectStr.outcome.fail"); + + std::string content_out; + fiu_enable("S3ClientWrapper.GetObjectStr.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.GetObjectStr(objname, content_out).ok()); + fiu_disable("S3ClientWrapper.GetObjectStr.outcome.fail"); + + ASSERT_FALSE(storage_inst.GetObjectStr(objname_dummy, content_out).ok()); + } + + /////////////////////////////////////////////////////////////////////////// + fiu_enable("S3ClientWrapper.DeleteObject.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.DeleteObject(filename).ok()); + fiu_disable("S3ClientWrapper.DeleteObject.outcome.fail"); + + fiu_enable("S3ClientWrapper.ListObjects.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.DeleteObjects("/tmp").ok()); + fiu_disable("S3ClientWrapper.ListObjects.outcome.fail"); + ASSERT_TRUE(storage_inst.DeleteObjects("/tmp").ok()); + + fiu_enable("S3ClientWrapper.DeleteBucket.outcome.fail", 1, NULL, 0); + ASSERT_FALSE(storage_inst.DeleteBucket().ok()); + fiu_disable("S3ClientWrapper.DeleteBucket.outcome.fail"); + + storage_inst.StopService(); +} diff --git a/core/unittest/storage/utils.cpp b/core/unittest/storage/utils.cpp index bcc76c0bed..4d80416bb7 100644 --- a/core/unittest/storage/utils.cpp +++ b/core/unittest/storage/utils.cpp @@ -18,6 +18,7 @@ #include #include +#include "server/Config.h" #include "storage/utils.h" #include "utils/CommonUtil.h" @@ -50,7 +51,11 @@ void StorageTest::SetUp() { std::string config_path(CONFIG_PATH); milvus::server::CommonUtil::CreateDirectory(config_path); - WriteToFile(config_path + CONFIG_FILE, CONFIG_STR); + config_path += CONFIG_FILE; + WriteToFile(config_path, CONFIG_STR); + + milvus::server::Config& config = milvus::server::Config::GetInstance(); + ASSERT_TRUE(config.LoadConfigFile(config_path).ok()); } void diff --git a/docker-compose.yml b/docker-compose.yml index 59b3e7ce46..4e9b1ec0c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,7 @@ services: - milvus command: &ubuntu-command > /bin/bash -c " - /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" centos-core: @@ -60,7 +60,7 @@ services: - milvus command: ¢os-command > /bin/bash --login -c " - /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" networks: From 95be87026ab8f6aeb7cc069ae5a120715050df4e Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 8 Jan 2020 10:40:30 +0800 Subject: [PATCH 10/14] Create SECURITY.md (#947) --- SECURITY.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..861969fcd3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +## Supported versions + +The following versions of Milvus are currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 0.6.0 | ✔️ | +| <= 0.5.3 | :x: | + +## Reporting a vulnerability + +To report a security vulnerability, please reach out to the Milvus team via . + From 4a116a6018693c87c2a8bf65db09d4c2ea76da98 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Wed, 8 Jan 2020 14:51:58 +0800 Subject: [PATCH 11/14] [skip ci] Create SUPPORT.md and RELEASE.md (#950) * [skip ci] Create SUPPORT.md * [skip ci] Create RELEASE.md --- RELEASE.md | 17 +++++++++++++++++ SUPPORT.md | 14 ++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 RELEASE.md create mode 100644 SUPPORT.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000000..afb0b63f50 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,17 @@ +# Milvus Release Methodology and Criterias + +## Release methodology + +Milvus releases are packages that have been approved for general public release, with varying degrees of caveat regarding their perceived quality or potential for change. +They are stable releases intended for everyday usage by developers and non-developers. + +Project versioning follows the specification of [Semantic Versioning 2.0.0](https://semver.org/). + +## Release criteria + +- Milvus core test code coverage must be at least 90%. +- Reported bugs should not have any critical issues. +- All bugs, new features, enhancements must be tested. +- All documents need to be reviewed with no broken link. +- Pressure testing, stability testing, accuracy testing and performance testing results should be evaluated. + diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000000..abce24a0bf --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,14 @@ +# Support for deploying and using Milvus + +We use GitHub for tracking bugs and feature requests. If you need any support for using Milvus, please refer to the following resources below. + +## Documentation +- [User Documentation](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) +- [Troubleshooting Guide](https://www.milvus.io/docs/v0.6.0/guides/troubleshoot.md) +- [FAQ](https://www.milvus.io/docs/v0.6.0/faq/operational_faq.md) + +## Real-time chat +[Slack](https://join.slack.com/t/milvusio/shared_invite/enQtNzY1OTQ0NDI3NjMzLWNmYmM1NmNjOTQ5MGI5NDhhYmRhMGU5M2NhNzhhMDMzY2MzNDdlYjM5ODQ5MmE3ODFlYzU3YjJkNmVlNDQ2ZTk): The #general channel is the place where people offer support. + +## Other +[Bootcamp](https://github.com/milvus-io/bootcamp): It provides more scenario-based applications and demos of Milvus. From c920ceb65b1c33d6ba79e48301e3bbe6791090a7 Mon Sep 17 00:00:00 2001 From: jielinxu <52057195+jielinxu@users.noreply.github.com> Date: Thu, 9 Jan 2020 11:39:33 +0800 Subject: [PATCH 12/14] [skip ci] Fix some broken links (#960) * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken link * [skip ci] Fix broken links --- README.md | 2 +- README_CN.md | 2 +- README_JP.md | 2 +- docs/README.md | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9a4741d8b0..8530ce5534 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Keep up-to-date with newest releases and latest updates by reading Milvus [relea ## Get started -See the [Milvus install guide](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) for using Docker containers. To install Milvus from source code, see [build from source](install.md). +See the [Milvus install guide](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) for using Docker containers. To install Milvus from source code, see [build from source](INSTALL.md). To edit Milvus settings, read [Milvus configuration](https://www.milvus.io/docs/v0.6.0/reference/milvus_config.md). diff --git a/README_CN.md b/README_CN.md index d33efe1c5d..c50806119e 100644 --- a/README_CN.md +++ b/README_CN.md @@ -23,7 +23,7 @@ Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java]( ## 开始使用 Milvus -请参阅 [Milvus 安装指南](https://www.milvus.io/cn/docs/guides/get_started/install_milvus/install_milvus.md) 使用 Docker 容器安装 Milvus。若要基于源码编译，请访问 [源码安装](install.md)。 +请参阅 [Milvus 安装指南](https://www.milvus.io/cn/docs/guides/get_started/install_milvus/install_milvus.md) 使用 Docker 容器安装 Milvus。若要基于源码编译，请访问 [源码安装](INSTALL.md)。 若要更改 Milvus 设置，请参阅 [Milvus 配置](https://www.milvus.io/cn/docs/reference/milvus_config.md)。 diff --git a/README_JP.md b/README_JP.md index 685ef6eddb..b09f2d3f32 100644 --- a/README_JP.md +++ b/README_JP.md @@ -23,7 +23,7 @@ Milvus [リリースノート](https://www.milvus.io/docs/v0.6.0/releases/v0.6.0 ## はじめに -DockerでMilvusをインストールすることは簡単です。[Milvusインストール案内](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) を参考してください。ソースからMilvusを構築するために、[ソースから構築する](install.md)を参考してください。 +DockerでMilvusをインストールすることは簡単です。[Milvusインストール案内](https://www.milvus.io/docs/guides/get_started/install_milvus/install_milvus.md) を参考してください。ソースからMilvusを構築するために、[ソースから構築する](INSTALL.md)を参考してください。 Milvusをコンフィグするために、[Milvusコンフィグ](https://www.milvus.io/docs/reference/milvus_config.md)を読んでください。 diff --git a/docs/README.md b/docs/README.md index 51f9f556f0..f149f6a3aa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,10 +16,10 @@ The following is a list of existing test reports: - [IVF_SQ8](test_report/milvus_ivfsq8_test_report_detailed_version.md) - [IVF_SQ8H](test_report/milvus_ivfsq8h_test_report_detailed_version.md) -- [IVFLAT](test-report/ivfflat_test_report_en.md) +- [IVFLAT](test_report/ivfflat_test_report_en.md) To read the CN version of these reports: - [IVF_SQ8_cn](test_report/milvus_ivfsq8_test_report_detailed_version_cn.md) - [IVF_SQ8H_cn](test_report/milvus_ivfsq8h_test_report_detailed_version_cn.md) -- [IVFLAT_cn](test-report/ivfflat_test_report_cn.md) +- [IVFLAT_cn](test_report/ivfflat_test_report_cn.md) From bdd9fc878b368831321cb849f45be6141d2de499 Mon Sep 17 00:00:00 2001 From: JackLCL <53512883+JackLCL@users.noreply.github.com> Date: Thu, 9 Jan 2020 15:34:25 +0800 Subject: [PATCH 13/14] fix issue 373 (#964) * fix issue 373 * Adjustment format * Adjustment format * Adjustment format * change readme --- core/src/server/DBWrapper.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index e0b71205df..2f299f82c2 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -165,7 +165,8 @@ DBWrapper::StartService() { db_ = engine::DBFactory::Build(opt); } catch (std::exception& ex) { std::cerr << "Error: failed to open database: " << ex.what() - << ". Possible reason: the meta system does not work." << std::endl; + << ". Possible reason: Meta Tables schema is damaged " + << "or created by in-compatible Milvus version." << std::endl; kill(0, SIGUSR1); } From b2f19ace0e1dcd431a512141f42b748581d4b92d Mon Sep 17 00:00:00 2001 From: Cai Yudong Date: Thu, 9 Jan 2020 15:35:13 +0800 Subject: [PATCH 14/14] #966 update NOTICE.md (#967) --- CHANGELOG.md | 1 + NOTICE.md | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 711614e600..8fb8d0c26e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ Please mark all change in change log and use the issue from GitHub - \#860 - Remove redundant checks in CacheMgr's constructor - \#908 - Move "primary_path" and "secondary_path" to storage config - \#931 - Remove "collector" from config +- \#966 - Update NOTICE.md ## Task diff --git a/NOTICE.md b/NOTICE.md index fee2f37a7a..a05688e6ca 100644 --- a/NOTICE.md +++ b/NOTICE.md @@ -23,4 +23,7 @@ | grpc | [Apache 2.0](https://github.com/grpc/grpc/blob/master/LICENSE) | | EASYLOGGINGPP | [MIT](https://github.com/zuhd-org/easyloggingpp/blob/master/LICENSE) | | Json | [MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT) | +| opentracing-cpp | [Apache License 2.0](https://github.com/opentracing/opentracing-cpp/blob/master/LICENSE) | +| libfiu | [BOLA](https://github.com/albertito/libfiu/blob/master/LICENSE) | +| aws-sdk-cpp | [Apache License 2.0](https://github.com/cydrain/aws-sdk-cpp/blob/master/LICENSE) |