mirror of https://github.com/milvus-io/milvus.git
MS-555 Remove old scheduler
Former-commit-id: e07f211e59232b04cfa3f37240e896461ad9b616pull/191/head
wxyu
commit
3c6f49f9cc
|
|
@ -13,12 +13,15 @@ Please mark all change in change log and use the ticket from JIRA.
|
|||
- MS-556 - Add Job Definition in Scheduler
|
||||
- MS-558 - Refine status code
|
||||
- MS-562 - Add JobMgr and TaskCreator in Scheduler
|
||||
- MS-566 - Refactor cmake
|
||||
- MS-555 - Remove old scheduler
|
||||
|
||||
## New Feature
|
||||
|
||||
## Task
|
||||
- MS-554 - Change license to Apache 2.0
|
||||
- MS-561 - Add contributing guidelines, code of conduct and README docs
|
||||
- MS-567 - Add NOTICE.md
|
||||
|
||||
# Milvus 0.4.0 (2019-09-12)
|
||||
|
||||
|
|
|
|||
|
|
@ -42,24 +42,15 @@ set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
|
|||
string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
|
||||
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||
set(BUILD_TYPE "release")
|
||||
set(BUILD_TYPE "Release")
|
||||
else()
|
||||
set(BUILD_TYPE "debug")
|
||||
set(BUILD_TYPE "Debug")
|
||||
endif()
|
||||
message(STATUS "Build type = ${BUILD_TYPE}")
|
||||
|
||||
add_definitions(-DNEW_SCHEDULER)
|
||||
|
||||
project(milvus VERSION "${MILVUS_VERSION}")
|
||||
project(milvus_engine LANGUAGES CUDA CXX)
|
||||
|
||||
# Ensure that a default make is set
|
||||
if("${MAKE}" STREQUAL "")
|
||||
if(NOT MSVC)
|
||||
find_program(MAKE make)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set(MILVUS_VERSION_MAJOR "${milvus_VERSION_MAJOR}")
|
||||
set(MILVUS_VERSION_MINOR "${milvus_VERSION_MINOR}")
|
||||
set(MILVUS_VERSION_PATCH "${milvus_VERSION_PATCH}")
|
||||
|
|
@ -78,86 +69,78 @@ message(STATUS "Milvus version: "
|
|||
"${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "
|
||||
"(full: '${MILVUS_VERSION}')")
|
||||
|
||||
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
|
||||
|
||||
find_package(CUDA)
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES -arch sm_60 --expt-extended-lambda")
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
|
||||
message(STATUS "CUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}")
|
||||
message(STATUS "CUDA_NVCC_FLAGS=${CUDA_NVCC_FLAGS}")
|
||||
|
||||
set(CMAKE_CXX_STANDARD 14)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED on)
|
||||
|
||||
if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
|
||||
message(STATUS "building milvus_engine on x86 architecture")
|
||||
message(STATUS "Building milvus_engine on x86 architecture")
|
||||
set(MILVUS_BUILD_ARCH x86_64)
|
||||
elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(ppc)")
|
||||
message(STATUS "building milvus_engine on ppc architecture")
|
||||
message(STATUS "Building milvus_engine on ppc architecture")
|
||||
set(MILVUS_BUILD_ARCH ppc64le)
|
||||
else()
|
||||
message(WARNING "unknown processor type")
|
||||
message(WARNING "Unknown processor type")
|
||||
message(WARNING "CMAKE_SYSTEM_PROCESSOR=${CMAKE_SYSTEM_PROCESSOR}")
|
||||
set(MILVUS_BUILD_ARCH unknown)
|
||||
endif()
|
||||
|
||||
find_package(CUDA)
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -Xcompiler -fPIC -std=c++11 -D_FORCE_INLINES --expt-extended-lambda")
|
||||
|
||||
if(CMAKE_BUILD_TYPE STREQUAL "Release")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -fPIC -DELPP_THREAD_SAFE -fopenmp")
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp")
|
||||
set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g")
|
||||
endif()
|
||||
|
||||
set(ALLOW_DUPLICATE_CUSTOM_TARGETS TRUE)
|
||||
# Ensure that a default make is set
|
||||
if("${MAKE}" STREQUAL "")
|
||||
if(NOT MSVC)
|
||||
find_program(MAKE make)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
find_path(MYSQL_INCLUDE_DIR
|
||||
NAMES "mysql.h"
|
||||
PATH_SUFFIXES "mysql")
|
||||
if (${MYSQL_INCLUDE_DIR} STREQUAL "MYSQL_INCLUDE_DIR-NOTFOUND")
|
||||
message(FATAL_ERROR "Could not found MySQL include directory")
|
||||
else()
|
||||
include_directories(${MYSQL_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
|
||||
|
||||
set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
|
||||
set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
|
||||
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
|
||||
|
||||
include(ExternalProject)
|
||||
include(DefineOptions)
|
||||
include(BuildUtils)
|
||||
include(ThirdPartyPackages)
|
||||
|
||||
include_directories(${MILVUS_SOURCE_DIR})
|
||||
link_directories(${MILVUS_BINARY_DIR})
|
||||
|
||||
## Following should be check
|
||||
|
||||
set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include)
|
||||
set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
|
||||
|
||||
add_compile_definitions(PROFILER=${PROFILER})
|
||||
|
||||
message(STATUS "MILVUS_ENABLE_PROFILING = ${MILVUS_ENABLE_PROFILING}")
|
||||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
ADD_DEFINITIONS(-DMILVUS_ENABLE_PROFILING)
|
||||
endif()
|
||||
|
||||
include_directories(${MILVUS_ENGINE_INCLUDE})
|
||||
include_directories(${MILVUS_ENGINE_SRC})
|
||||
|
||||
link_directories(${CMAKE_CURRRENT_BINARY_DIR})
|
||||
config_summary()
|
||||
|
||||
add_subdirectory(src)
|
||||
|
||||
if (BUILD_COVERAGE STREQUAL "ON")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
|
||||
endif()
|
||||
|
||||
|
||||
if (BUILD_UNIT_TEST STREQUAL "ON")
|
||||
if (BUILD_COVERAGE STREQUAL "ON")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
|
||||
endif()
|
||||
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest)
|
||||
endif()
|
||||
|
||||
add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean)
|
||||
|
||||
|
||||
if("${MILVUS_DB_PATH}" STREQUAL "")
|
||||
set(MILVUS_DB_PATH "/tmp/milvus")
|
||||
endif()
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml)
|
||||
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf)
|
||||
|
||||
#install
|
||||
install(DIRECTORY scripts/
|
||||
DESTINATION scripts
|
||||
FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ
|
||||
|
|
@ -169,12 +152,3 @@ install(FILES
|
|||
conf/log_config.conf
|
||||
DESTINATION
|
||||
conf)
|
||||
install(FILES
|
||||
./Milvus-EULA-cn.md
|
||||
./Milvus-EULA-en.md
|
||||
DESTINATION
|
||||
license
|
||||
)
|
||||
|
||||
|
||||
config_summary()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,48 @@
|
|||
# Milvus Code of Conduct
|
||||
|
||||
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Our Standards
|
||||
|
||||
Examples of behavior that contributes to creating a positive environment include:
|
||||
|
||||
- Using welcoming and inclusive language.
|
||||
- Being respectful of differing viewpoints and experiences.
|
||||
- Gracefully accepting constructive criticism.
|
||||
- Focusing on what is best for the community.
|
||||
- Showing empathy towards other community members.
|
||||
|
||||
Examples of unacceptable behavior by participants include:
|
||||
|
||||
- The use of sexualized language or imagery and unwelcome sexual attention or advances.
|
||||
- Trolling, insulting/derogatory comments, and personal or political attacks.
|
||||
- Public or private harassment.
|
||||
- Publishing others' private information, such as a physical or electronic address, without explicit permission.
|
||||
- Conduct which could reasonably be considered inappropriate for the forum in which it occurs.
|
||||
|
||||
All Milvus forums and spaces are meant for professional interactions, and any behavior which could reasonably be considered inappropriate in a professional setting is unacceptable.
|
||||
|
||||
## Our Responsibilities
|
||||
|
||||
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
|
||||
|
||||
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
|
||||
|
||||
## Scope
|
||||
|
||||
This Code of Conduct applies to all content on milvus.io, Milvus’s GitHub organization, or any other official Milvus web presence allowing for community interactions, as well as at all official Milvus events, whether offline or online.
|
||||
|
||||
The Code of Conduct also applies within all project spaces and in public spaces whenever an individual is representing Milvus or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed or de facto representative at an online or offline event. Representation of Milvus may be further defined and clarified by project maintainers.
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at support@zilliz.com. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
|
||||
|
||||
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/), version 1.4, available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
|
||||
|
||||
For answers to common questions about this code of conduct, see https://www.contributor-covenant.org/faq
|
||||
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
# Contributing to Milvus
|
||||
|
||||
First of all, thanks for taking the time to contribute to Milvus! It's people like you that help Milvus come to fruition.
|
||||
|
||||
The following are a set of guidelines for contributing to Milvus. Following these guidelines helps contributing to this project easy and transparent. These are mostly guideline, not rules. Use your best judgment, and feel free to propose changes to this document in a pull request.
|
||||
|
||||
As for everything else in the project, the contributions to Milvus are governed by our [Code of Conduct](http://hood.ie/code-of-conduct/).
|
||||
|
||||
TOC
|
||||
|
||||
## Contribution Checklist
|
||||
|
||||
Before you make any contributions, make sure you follow this list.
|
||||
|
||||
- Read [Contributing to Milvus](CONTRIBUTING.md).
|
||||
- Check if the changes are consistent with the [coding style](CONTRIBUTING.md#coding-style).
|
||||
- Run [unit tests](CONTRIBUTING.md#run-unit-test).
|
||||
|
||||
## What contributions can I make?
|
||||
|
||||
Contributions to Milvus fall into the following categories.
|
||||
|
||||
1. To report a bug or a problem with documentation, please file an [issue](https://github.com/milvus-io/milvus/issues/new) providing the details of the problem. If you believe the issue needs priority attention, please comment on the issue to notify the team.
|
||||
2. To propose a new feature, please file a new feature request [issue](https://github.com/milvus-io/milvus/issues/new). Describe the intended feature and discuss the design and implementation with the team and community. Once the team agrees that the plan looks good, go ahead and implement it, following the [Contributing code].
|
||||
3. To implement a feature or bug-fix for an existing outstanding issue, follow the [Contributing code]. If you need more context on a particular issue, comment on the issue to let people know.
|
||||
|
||||
## How can I contribute?
|
||||
|
||||
### Contributing code
|
||||
|
||||
If you have improvements to Milvus, send us your pull requests! For those just getting started, GitHub has a [how-to](https://help.github.com/en/articles/about-pull-requests).
|
||||
|
||||
The Milvus team members will review your pull requests, and once it is accepted, it will be given a `ready to merge` label. This means we are working on submitting your pull request to the internal repository. After the change has been submitted internally, your pull request will be merged automatically on GitHub.
|
||||
|
||||
### General guidelines
|
||||
|
||||
Before sending your pull requests for review, make sure your changes are consistent with the guidelines and follow the Milvus coding style.
|
||||
|
||||
- Include unit tests when you contribute new features, as they help to a) prove that your code works correctly, and b) guard against future breaking changes to lower the maintenance cost.
|
||||
- Bug fixes also generally require unit tests, because the presence of bugs usually indicates insufficient test coverage.
|
||||
- Keep API compatibility in mind when you change code in Milvus. Reviewers of your pull request will comment on any API compatibility issues.
|
||||
- When you contribute a new feature to Milvus, the maintenance burden is (by default) transferred to the Milvus team. This means that the benefit of the contribution must be compared against the cost of maintaining the feature.
|
||||
|
||||
|
||||
## Coding Style
|
||||
|
||||
## Run unit test
|
||||
|
||||
```shell
|
||||
$ ./build.sh -u
|
||||
or
|
||||
$ ./build.sh --unittest
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
| Name | License |
|
||||
| ------------- | ------------------------------------------------------------ |
|
||||
| Apache Arrow | Apache License 2.0 |
|
||||
| Boost | Boost Software License |
|
||||
| BZIP2 | BSD-like license |
|
||||
| FAISS | MIT |
|
||||
| Gtest | BSD 3-Clause |
|
||||
| LAPACK | BSD 3-Clause |
|
||||
| LZ4 | [BSD 2-Clause](https://github.com/Blosc/c-blosc/blob/master/LICENSES/LZ4.txt) |
|
||||
| MySQLPP | LGPL 2.1 |
|
||||
| OpenBLAS | BSD 3-Clause |
|
||||
| Prometheus | Apache License 2.0 |
|
||||
| Snappy | [BSD](https://github.com/Blosc/c-blosc/blob/master/LICENSES/SNAPPY.txt) |
|
||||
| SQLite | [Public Domain](https://www.sqlite.org/copyright.html) |
|
||||
| SQLite-ORM | BSD 3-Clause |
|
||||
| yaml-cpp | MIT |
|
||||
| ZLIB | [zlib](http://zlib.net/zlib_license.html) |
|
||||
| ZSTD | [BSD](https://github.com/facebook/zstd/blob/dev/LICENSE) |
|
||||
| libunwind | MIT |
|
||||
| gperftools | BSD 3-Clause |
|
||||
| gRPC | Apache 2.0 |
|
||||
| EASYLOGGINGPP | MIT |
|
||||
|
||||
245
cpp/README.md
245
cpp/README.md
|
|
@ -1,72 +1,211 @@
|
|||
### Compilation
|
||||
#### Step 1: install necessery tools
|
||||
# Welcome to Milvus
|
||||
|
||||
centos7 :
|
||||
yum install gfortran qt4 flex bison mysql-devel mysql
|
||||
|
||||
ubuntu16.04 :
|
||||
sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
|
||||
|
||||
cd scripts && sudo ./requirements.sh
|
||||
Firstly, welcome, and thanks for your interest in [Milvus](https://milvus.io)! No matter who you are, what you do, we greatly appreciate your contribution to help us reinvent data science with Milvus.
|
||||
|
||||
If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link:
|
||||
## What is Milvus
|
||||
|
||||
```
|
||||
sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
|
||||
Milvus is an open source vector search engine that makes incredibly fast querying speed enhancement over current solutions of massive vector processing. Built on optimized indexing algorithm, it is compatible with major AI/ML models.
|
||||
|
||||
Milvus was developed by researchers and engineers in ZILLIZ, a tech startup that intends to reinvent data science, with the purpose of providing enterprises with efficient and scalable similarity search and analysis of feature vectors and unstructured data.
|
||||
|
||||
Milvus provides stable Python and C++ APIs, as well as RESTful API.
|
||||
|
||||
Keep up-to-date with newest releases and latest updates by reading [Releases](https://www.milvus-io/docs/master/releases).
|
||||
|
||||
- GPU-accelerated search engine
|
||||
|
||||
Milvus is designed for the largest scale of vector index. CPU/GPU heterogeneous computing architecture allows you to process data at a speed 1000 times faster.
|
||||
|
||||
- Intelligent index
|
||||
|
||||
With a “Decide Your Own Algorithm” approach, you can embed machine learning and advanced algorithms into Milvus without the headache of complex data engineering or migrating data between disparate systems. Milvus is built on optimized indexing algorithm based on quantization indexing, tree-based and graph indexing methods.
|
||||
|
||||
- Strong scalability
|
||||
|
||||
The data is stored and computed on a distributed architecture. This lets you scale data sizes up and down without redesigning the system.
|
||||
|
||||
## Architecture
|
||||
|
||||
|
||||
## Get started
|
||||
|
||||
### Install and start Milvus server
|
||||
|
||||
#### Use Docker
|
||||
|
||||
Use Docker to install Milvus is a breeze. See the [Milvus install guide](https://www.milvus-io/docs/master/userguide/install_milvus.md) for details.
|
||||
|
||||
#### Use source code
|
||||
|
||||
##### Compilation
|
||||
|
||||
###### Step 1 Install necessary tools
|
||||
|
||||
```shell
|
||||
# Install tools
|
||||
Centos7 :
|
||||
$ yum install gfortran qt4 flex bison mysql-devel mysql
|
||||
|
||||
Ubuntu16.04 :
|
||||
$ sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client
|
||||
|
||||
```
|
||||
|
||||
#### Step 2: build(output to cmake_build folder)
|
||||
Verify the existence of `libmysqlclient_r.so`:
|
||||
|
||||
cmake_build/src/milvus_server is the server
|
||||
```shell
|
||||
# Verify existence
|
||||
$ locate libmysqlclient_r.so
|
||||
```
|
||||
|
||||
cd [sourcecode path]/cpp/thirdparty
|
||||
git clone git@192.168.1.105:megasearch/knowhere.git
|
||||
cd knowhere
|
||||
./build.sh -t Debug
|
||||
or ./build.sh -t Release
|
||||
If not, you need to create a symbolic link:
|
||||
|
||||
cd [sourcecode path]/cpp
|
||||
./build.sh -t Debug
|
||||
or ./build.sh -t Release
|
||||
```shell
|
||||
# Create symbolic link
|
||||
$ sudo ln -s /path/to/libmysqlclient.so /path/to/libmysqlclient_r.so
|
||||
```
|
||||
|
||||
If you encounter the following error when building:
|
||||
###### Step 2 Build
|
||||
|
||||
```shell
|
||||
TBD
|
||||
cd [Milvus sourcecode path]/cpp/thirdparty
|
||||
git clone git@192.168.1.105:megasearch/knowhere.git
|
||||
cd knowhere
|
||||
./build.sh -t Debug
|
||||
or ./build.sh -t Release
|
||||
|
||||
cd [sourcecode path]/cpp
|
||||
./build.sh -t Debug
|
||||
or ./build.sh -t Release
|
||||
```
|
||||
|
||||
When the build is completed, all the stuff that you need in order to run Milvus will be installed under `[Milvus root path]/cpp/milvus`.
|
||||
|
||||
If you encounter the following error message,
|
||||
`protocol https not supported or disabled in libcurl`
|
||||
|
||||
1. Install libcurl4-openssl-dev
|
||||
please reinstall CMake with curl:
|
||||
|
||||
2. Install cmake 3.14:
|
||||
|
||||
```
|
||||
./bootstrap --system-curl
|
||||
make
|
||||
sudo make install
|
||||
1. Install curl development files:
|
||||
```shell
|
||||
CentOS 7:
|
||||
$ yum install curl-devel
|
||||
Ubuntu 16.04:
|
||||
$ sudo apt-get install libcurl4-openssl-dev
|
||||
```
|
||||
|
||||
#### To build unittest:
|
||||
2. Install [CMake 3.14](https://github.com/Kitware/CMake/releases/download/v3.14.6/cmake-3.14.6.tar.gz):
|
||||
```shell
|
||||
$ ./bootstrap --system-curl
|
||||
$ make
|
||||
$ sudo make install
|
||||
```
|
||||
|
||||
./build.sh -u
|
||||
or
|
||||
./build.sh --unittest
|
||||
##### Run unit test
|
||||
|
||||
#### To run code coverage
|
||||
|
||||
apt-get install lcov
|
||||
./build.sh -u -c
|
||||
|
||||
### Launch server
|
||||
Set config in cpp/conf/server_config.yaml
|
||||
|
||||
Add milvus/lib to LD_LIBRARY_PATH
|
||||
|
||||
```
|
||||
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib
|
||||
```shell
|
||||
$ ./build.sh -u
|
||||
or
|
||||
$ ./build.sh --unittest
|
||||
```
|
||||
|
||||
Then launch server with config:
|
||||
cd [build output path]
|
||||
start_server.sh
|
||||
stop_server.sh
|
||||
##### Run code coverage
|
||||
|
||||
### Launch test_client(only for debug)
|
||||
If you want to test remote api, you can run sdk example.
|
||||
[build output path]/sdk/examples/grpcsimple/sdk_simple
|
||||
```shell
|
||||
CentOS 7:
|
||||
$ yum install lcov
|
||||
Ubuntu 16.04:
|
||||
$ sudo apt-get install lcov
|
||||
|
||||
$ ./build.sh -u -c
|
||||
```
|
||||
|
||||
##### Launch Milvus server
|
||||
|
||||
```shell
|
||||
$ cd [Milvus root path]/cpp/milvus
|
||||
```
|
||||
|
||||
Add `lib/` directory to `LD_LIBRARY_PATH`
|
||||
|
||||
```
|
||||
$ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib
|
||||
```
|
||||
|
||||
Then start Milvus server:
|
||||
|
||||
```
|
||||
$ cd scripts
|
||||
$ ./start_server.sh
|
||||
```
|
||||
|
||||
To stop Milvus server, run:
|
||||
|
||||
```shell
|
||||
$ ./stop_server.sh
|
||||
```
|
||||
|
||||
To edit Milvus settings in `conf/server_config.yaml` and `conf/log_config.conf`, please read [Milvus Configuration](https://www.milvus-io/docs/master/reference/milvus_config.md).
|
||||
|
||||
### Try your first Milvus program
|
||||
|
||||
#### Run Python example code
|
||||
|
||||
Make sure [Python 3.4](https://www.python.org/downloads/) or higher is already installed and in use.
|
||||
|
||||
Install Milvus Python SDK.
|
||||
|
||||
```shell
|
||||
# Install Milvus Python SDK
|
||||
$ pip install pymilvus==0.2.0
|
||||
```
|
||||
|
||||
Create a new file `example.py`, and add [Python example code](https://github.com/milvus-io/pymilvus/blob/branch-0.3.1/examples/AdvancedExample.py) to it.
|
||||
|
||||
Run the example code.
|
||||
|
||||
```python
|
||||
# Run Milvus Python example
|
||||
$ python3 example.py
|
||||
```
|
||||
|
||||
#### Run C++ example code
|
||||
|
||||
```shell
|
||||
# Run Milvus C++ example
|
||||
$ cd [Milvus root path]/cpp/milvus/bin
|
||||
$ ./sdk_simple
|
||||
```
|
||||
|
||||
## Contribution guidelines
|
||||
|
||||
Contributions are welcomed and greatly appreciated. If you want to contribute to Milvus, please read the [contribution guidelines](CONTRIBUTING.md). This project adheres to the [code of conduct](CODE OF CONDUCT.md) of Milvus. By participating, you are expected to uphold this code.
|
||||
|
||||
We use [GitHub issues] to track issues and bugs. For general questions and discussions, please go to [Milvus Forum].
|
||||
|
||||
## Join the Milvus community
|
||||
|
||||
For public discussion of Milvus, please join our [discussion group].
|
||||
|
||||
## Milvus Roadmap
|
||||
|
||||
Please read our [roadmap](milvus-io/milvus/docs/master/roadmap.md) to learn about upcoming features.
|
||||
|
||||
## Resources
|
||||
|
||||
[Milvus official website](https://milvus.io)
|
||||
|
||||
[Milvus docs](https://www.milvus.io/docs/en/QuickStart/)
|
||||
|
||||
[Milvus blog](https://www.milvus.io/blog/)
|
||||
|
||||
[Milvus CSDN](https://mp.csdn.net/mdeditor/100041006#)
|
||||
|
||||
[Milvus roadmap](https://www.milvus-io/docs/master/roadmap.md)
|
||||
|
||||
[Milvus white paper]
|
||||
|
||||
## License
|
||||
|
||||
[Apache 2.0 license](milvus-io/milvus/LICENSE.md)
|
||||
|
|
@ -55,8 +55,6 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
|
|||
define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
|
||||
"Show output from ExternalProjects rather than just logging to files" ON)
|
||||
|
||||
define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF)
|
||||
|
||||
define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \
|
||||
Note that this requires linking Boost statically" ON)
|
||||
|
||||
|
|
@ -66,22 +64,10 @@ define_option(MILVUS_WITH_BZ2 "Build with BZ2 compression" ON)
|
|||
|
||||
define_option(MILVUS_WITH_EASYLOGGINGPP "Build with Easylogging++ library" ON)
|
||||
|
||||
define_option(MILVUS_WITH_FAISS "Build with FAISS library" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_LAPACK "Build with LAPACK library" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_LZ4 "Build with lz4 compression" ON)
|
||||
|
||||
define_option(MILVUS_WITH_JSONCONS "Build with JSONCONS" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_OPENBLAS "Build with OpenBLAS library" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_PROMETHEUS "Build with PROMETHEUS library" ON)
|
||||
|
||||
define_option(MILVUS_WITH_ROCKSDB "Build with RocksDB library" OFF)
|
||||
|
||||
define_option(MILVUS_WITH_SNAPPY "Build with Snappy compression" ON)
|
||||
|
||||
define_option(MILVUS_WITH_SQLITE "Build with SQLite library" ON)
|
||||
|
|
@ -94,10 +80,6 @@ define_option(MILVUS_WITH_YAMLCPP "Build with yaml-cpp library" ON)
|
|||
|
||||
define_option(MILVUS_WITH_ZLIB "Build with zlib compression" ON)
|
||||
|
||||
define_option(MILVUS_WITH_KNOWHERE "Build with Knowhere" OFF)
|
||||
|
||||
#define_option(MILVUS_ENABLE_PROFILING "Build with profiling" ON)
|
||||
|
||||
if(CMAKE_VERSION VERSION_LESS 3.7)
|
||||
set(MILVUS_WITH_ZSTD_DEFAULT OFF)
|
||||
else()
|
||||
|
|
@ -106,13 +88,13 @@ else()
|
|||
endif()
|
||||
define_option(MILVUS_WITH_ZSTD "Build with zstd compression" ${MILVUS_WITH_ZSTD_DEFAULT})
|
||||
|
||||
define_option(MILVUS_WITH_AWS "Build with AWS SDK" ON)
|
||||
|
||||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
define_option(MILVUS_WITH_LIBUNWIND "Build with libunwind" ON)
|
||||
define_option(MILVUS_WITH_GPERFTOOLS "Build with gperftools" ON)
|
||||
endif()
|
||||
|
||||
define_option(MILVUS_WITH_GRPC "Build with GRPC" ON)
|
||||
|
||||
#----------------------------------------------------------------------
|
||||
if(MSVC)
|
||||
set_option_category("MSVC")
|
||||
|
|
|
|||
|
|
@ -16,27 +16,18 @@
|
|||
|
||||
set(MILVUS_THIRDPARTY_DEPENDENCIES
|
||||
|
||||
ARROW
|
||||
BOOST
|
||||
BZip2
|
||||
FAISS
|
||||
GTest
|
||||
Knowhere
|
||||
JSONCONS
|
||||
LAPACK
|
||||
Lz4
|
||||
MySQLPP
|
||||
OpenBLAS
|
||||
Prometheus
|
||||
RocksDB
|
||||
Snappy
|
||||
SQLite
|
||||
SQLite_ORM
|
||||
Thrift
|
||||
yaml-cpp
|
||||
ZLIB
|
||||
ZSTD
|
||||
AWS
|
||||
libunwind
|
||||
gperftools
|
||||
GRPC)
|
||||
|
|
@ -51,30 +42,16 @@ foreach(DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES})
|
|||
endforeach()
|
||||
|
||||
macro(build_dependency DEPENDENCY_NAME)
|
||||
if("${DEPENDENCY_NAME}" STREQUAL "ARROW")
|
||||
build_arrow()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "BZip2")
|
||||
if("${DEPENDENCY_NAME}" STREQUAL "BZip2")
|
||||
build_bzip2()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "FAISS")
|
||||
build_faiss()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
|
||||
build_gtest()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "LAPACK")
|
||||
build_lapack()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "Knowhere")
|
||||
build_knowhere()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "Lz4")
|
||||
build_lz4()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "MySQLPP")
|
||||
build_mysqlpp()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "JSONCONS")
|
||||
build_jsoncons()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
|
||||
build_openblas()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "Prometheus")
|
||||
build_prometheus()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "RocksDB")
|
||||
build_rocksdb()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "Snappy")
|
||||
build_snappy()
|
||||
elseif ("${DEPENDENCY_NAME}" STREQUAL "SQLite")
|
||||
|
|
@ -87,8 +64,6 @@ macro(build_dependency DEPENDENCY_NAME)
|
|||
build_zlib()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "ZSTD")
|
||||
build_zstd()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "AWS")
|
||||
build_aws()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "libunwind")
|
||||
build_libunwind()
|
||||
elseif("${DEPENDENCY_NAME}" STREQUAL "gperftools")
|
||||
|
|
@ -280,8 +255,7 @@ if(DEFINED ENV{MILVUS_BOOST_URL})
|
|||
else()
|
||||
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORES ${BOOST_VERSION})
|
||||
set(BOOST_SOURCE_URL
|
||||
"http://192.168.1.201/artifactory/generic-local/tools/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz")
|
||||
# "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz"
|
||||
"https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz")
|
||||
endif()
|
||||
set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26")
|
||||
|
||||
|
|
@ -292,22 +266,6 @@ else()
|
|||
endif()
|
||||
set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b")
|
||||
|
||||
if(DEFINED ENV{MILVUS_FAISS_URL})
|
||||
set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}")
|
||||
else()
|
||||
set(FAISS_SOURCE_URL "http://192.168.1.105:6060/jinhai/faiss/-/archive/${FAISS_VERSION}/faiss-${FAISS_VERSION}.tar.gz")
|
||||
# set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz")
|
||||
endif()
|
||||
|
||||
# set(FAISS_MD5 "a589663865a8558205533c8ac414278c")
|
||||
set(FAISS_MD5 "31167ecbd1903fec600dc4ac00b9be9e")
|
||||
|
||||
if(DEFINED ENV{MILVUS_KNOWHERE_URL})
|
||||
set(KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}")
|
||||
else()
|
||||
set(KNOWHERE_SOURCE_URL "${CMAKE_SOURCE_DIR}/thirdparty/knowhere")
|
||||
endif()
|
||||
|
||||
if (DEFINED ENV{MILVUS_GTEST_URL})
|
||||
set(GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}")
|
||||
else ()
|
||||
|
|
@ -316,20 +274,6 @@ else ()
|
|||
endif()
|
||||
set(GTEST_MD5 "2e6fbeb6a91310a16efe181886c59596")
|
||||
|
||||
if (DEFINED ENV{MILVUS_JSONCONS_URL})
|
||||
set(JSONCONS_SOURCE_URL "$ENV{MILVUS_JSONCONS_URL}")
|
||||
else ()
|
||||
set(JSONCONS_SOURCE_URL
|
||||
"https://github.com/danielaparker/jsoncons/archive/v${JSONCONS_VERSION}.tar.gz")
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{MILVUS_LAPACK_URL})
|
||||
set(LAPACK_SOURCE_URL "$ENV{MILVUS_LAPACK_URL}")
|
||||
else()
|
||||
set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz")
|
||||
endif()
|
||||
set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2")
|
||||
|
||||
if(DEFINED ENV{MILVUS_LZ4_URL})
|
||||
set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}")
|
||||
else()
|
||||
|
|
@ -344,14 +288,6 @@ else()
|
|||
endif()
|
||||
set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79")
|
||||
|
||||
if (DEFINED ENV{MILVUS_OPENBLAS_URL})
|
||||
set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}")
|
||||
else ()
|
||||
set(OPENBLAS_SOURCE_URL
|
||||
"https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz")
|
||||
endif()
|
||||
set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495")
|
||||
|
||||
if (DEFINED ENV{MILVUS_PROMETHEUS_URL})
|
||||
set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}")
|
||||
else ()
|
||||
|
|
@ -359,15 +295,6 @@ else ()
|
|||
https://github.com/jupp0r/prometheus-cpp.git)
|
||||
endif()
|
||||
|
||||
|
||||
if (DEFINED ENV{MILVUS_ROCKSDB_URL})
|
||||
set(ROCKSDB_SOURCE_URL "$ENV{MILVUS_ROCKSDB_URL}")
|
||||
else ()
|
||||
set(ROCKSDB_SOURCE_URL
|
||||
"https://github.com/facebook/rocksdb/archive/${ROCKSDB_VERSION}.tar.gz")
|
||||
endif()
|
||||
set(ROCKSDB_MD5 "a8f2f594182e97a08629bcc66dfd3fa0")
|
||||
|
||||
if(DEFINED ENV{MILVUS_SNAPPY_URL})
|
||||
set(SNAPPY_SOURCE_URL "$ENV{MILVUS_SNAPPY_URL}")
|
||||
else()
|
||||
|
|
@ -413,13 +340,6 @@ else()
|
|||
endif()
|
||||
set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120")
|
||||
|
||||
if(DEFINED ENV{MILVUS_AWS_URL})
|
||||
set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}")
|
||||
else()
|
||||
set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz")
|
||||
endif()
|
||||
set(AWS_MD5 "9217f5bc8bf23dea04f4466521c85fd9")
|
||||
|
||||
if(DEFINED ENV{MILVUS_LIBUNWIND_URL})
|
||||
set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}")
|
||||
else()
|
||||
|
|
@ -445,120 +365,6 @@ endif()
|
|||
set(GRPC_MD5 "7ec59ad54c85a12dcbbfede09bf413a9")
|
||||
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# ARROW
|
||||
|
||||
macro(build_arrow)
|
||||
message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source")
|
||||
set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp")
|
||||
set(ARROW_STATIC_LIB_NAME arrow)
|
||||
|
||||
set(ARROW_STATIC_LIB
|
||||
"${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
|
||||
set(ARROW_CMAKE_ARGS
|
||||
${EP_COMMON_CMAKE_ARGS}
|
||||
-DARROW_BUILD_STATIC=ON
|
||||
-DARROW_BUILD_SHARED=OFF
|
||||
-DARROW_PARQUET=ON
|
||||
-DARROW_USE_GLOG=OFF
|
||||
-DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX}
|
||||
"-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs"
|
||||
-DCMAKE_BUILD_TYPE=Release)
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
execute_process(COMMAND sh -c "git ls-remote --heads --tags ${ARROW_SOURCE_URL} ${ARROW_VERSION} | cut -f 1" OUTPUT_VARIABLE ARROW_LAST_COMMIT_ID)
|
||||
if(${ARROW_LAST_COMMIT_ID} MATCHES "^[^#][a-z0-9]+")
|
||||
string(MD5 ARROW_COMBINE_MD5 "${ARROW_LAST_COMMIT_ID}")
|
||||
set(ARROW_CACHE_PACKAGE_NAME "arrow_${ARROW_COMBINE_MD5}.tar.gz")
|
||||
set(ARROW_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ARROW_CACHE_PACKAGE_NAME}")
|
||||
set(ARROW_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ARROW_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${ARROW_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote cache file ${ARROW_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(arrow_ep
|
||||
GIT_REPOSITORY
|
||||
${ARROW_SOURCE_URL}
|
||||
GIT_TAG
|
||||
${ARROW_VERSION}
|
||||
GIT_SHALLOW
|
||||
TRUE
|
||||
SOURCE_SUBDIR
|
||||
cpp
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${ARROW_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_COMMAND
|
||||
${MAKE} install
|
||||
BUILD_BYPRODUCTS
|
||||
"${ARROW_STATIC_LIB}"
|
||||
)
|
||||
|
||||
ExternalProject_Create_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ARROW_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${ARROW_CACHE_URL} ${ARROW_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${ARROW_CACHE_URL} TO ${ARROW_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(arrow_ep ${ARROW_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
message(FATAL_ERROR "The last commit ID of \"${ARROW_SOURCE_URL}\" repository don't match!")
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(arrow_ep
|
||||
GIT_REPOSITORY
|
||||
${ARROW_SOURCE_URL}
|
||||
GIT_TAG
|
||||
${ARROW_VERSION}
|
||||
GIT_SHALLOW
|
||||
TRUE
|
||||
SOURCE_SUBDIR
|
||||
cpp
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${ARROW_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_COMMAND
|
||||
${MAKE} install
|
||||
BUILD_BYPRODUCTS
|
||||
"${ARROW_STATIC_LIB}"
|
||||
)
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
|
||||
add_library(arrow STATIC IMPORTED)
|
||||
set_target_properties(arrow
|
||||
PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}")
|
||||
|
||||
add_dependencies(arrow arrow_ep)
|
||||
|
||||
set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
|
||||
|
||||
add_custom_command(TARGET arrow_ep POST_BUILD
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/
|
||||
DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
|
||||
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_ARROW)
|
||||
|
||||
resolve_dependency(ARROW)
|
||||
|
||||
link_directories(SYSTEM ${ARROW_PREFIX}/lib/)
|
||||
include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Add Boost dependencies (code adapted from Apache Kudu (incubating))
|
||||
|
||||
|
|
@ -755,370 +561,6 @@ if(MILVUS_WITH_BZ2)
|
|||
include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}")
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Knowhere
|
||||
|
||||
macro(build_knowhere)
|
||||
message(STATUS "Building knowhere from source")
|
||||
set(KNOWHERE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/knowhere_ep-prefix/src/knowhere_ep")
|
||||
set(KNOWHERE_INCLUDE_DIR "${KNOWHERE_PREFIX}/include")
|
||||
set(KNOWHERE_STATIC_LIB
|
||||
"${KNOWHERE_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}knowhere${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
|
||||
set(KNOWHERE_CMAKE_ARGS
|
||||
${EP_COMMON_CMAKE_ARGS}
|
||||
"-DCMAKE_INSTALL_PREFIX=${KNOWHERE_PREFIX}"
|
||||
-DCMAKE_INSTALL_LIBDIR=lib
|
||||
"-DCMAKE_CUDA_COMPILER=${CMAKE_CUDA_COMPILER}"
|
||||
"-DCUDA_TOOLKIT_ROOT_DIR=${CUDA_TOOLKIT_ROOT_DIR}"
|
||||
-DCMAKE_BUILD_TYPE=Release)
|
||||
|
||||
externalproject_add(knowhere_ep
|
||||
URL
|
||||
${KNOWHERE_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${KNOWHERE_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
BUILD_BYPRODUCTS
|
||||
${KNOWHERE_STATIC_LIB})
|
||||
|
||||
file(MAKE_DIRECTORY "${KNOWHERE_INCLUDE_DIR}")
|
||||
add_library(knowhere STATIC IMPORTED)
|
||||
set_target_properties(
|
||||
knowhere
|
||||
PROPERTIES IMPORTED_LOCATION "${KNOWHERE_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${KNOWHERE_INCLUDE_DIR}")
|
||||
|
||||
add_dependencies(knowhere knowhere_ep)
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_KNOWHERE)
|
||||
resolve_dependency(Knowhere)
|
||||
|
||||
get_target_property(KNOWHERE_INCLUDE_DIR knowhere INTERFACE_INCLUDE_DIRECTORIES)
|
||||
link_directories(SYSTEM "${KNOWHERE_PREFIX}/lib")
|
||||
include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}")
|
||||
include_directories(SYSTEM "${KNOWHERE_INCLUDE_DIR}/SPTAG/AnnService")
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# OpenBLAS
|
||||
|
||||
macro(build_openblas)
|
||||
message(STATUS "Building OpenBLAS-${OPENBLAS_VERSION} from source")
|
||||
set(OPENBLAS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix/src/openblas_ep")
|
||||
set(OPENBLAS_INCLUDE_DIR "${OPENBLAS_PREFIX}/include")
|
||||
set(OPENBLAS_STATIC_LIB
|
||||
"${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz")
|
||||
set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}")
|
||||
set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${OPENBLAS_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote file ${OPENBLAS_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(openblas_ep
|
||||
URL
|
||||
${OPENBLAS_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
""
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_COMMAND
|
||||
${MAKE}
|
||||
PREFIX=${OPENBLAS_PREFIX}
|
||||
install
|
||||
BUILD_BYPRODUCTS
|
||||
${OPENBLAS_STATIC_LIB})
|
||||
|
||||
ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(openblas_ep
|
||||
URL
|
||||
${OPENBLAS_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
""
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_COMMAND
|
||||
${MAKE}
|
||||
PREFIX=${OPENBLAS_PREFIX}
|
||||
install
|
||||
BUILD_BYPRODUCTS
|
||||
${OPENBLAS_STATIC_LIB})
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}")
|
||||
add_library(openblas STATIC IMPORTED)
|
||||
set_target_properties(
|
||||
openblas
|
||||
PROPERTIES IMPORTED_LOCATION "${OPENBLAS_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${OPENBLAS_INCLUDE_DIR}")
|
||||
|
||||
add_dependencies(openblas openblas_ep)
|
||||
endmacro()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# LAPACK
|
||||
|
||||
macro(build_lapack)
|
||||
message(STATUS "Building LAPACK-${LAPACK_VERSION} from source")
|
||||
set(LAPACK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix/src/lapack_ep")
|
||||
set(LAPACK_INCLUDE_DIR "${LAPACK_PREFIX}/include")
|
||||
set(LAPACK_STATIC_LIB
|
||||
"${LAPACK_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}lapack${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
|
||||
set(LAPACK_CMAKE_ARGS
|
||||
${EP_COMMON_CMAKE_ARGS}
|
||||
"-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}"
|
||||
-DCMAKE_INSTALL_LIBDIR=lib)
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz")
|
||||
set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}")
|
||||
set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${LAPACK_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote file ${LAPACK_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(lapack_ep
|
||||
URL
|
||||
${LAPACK_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${LAPACK_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
BUILD_BYPRODUCTS
|
||||
${LAPACK_STATIC_LIB})
|
||||
|
||||
ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(lapack_ep
|
||||
URL
|
||||
${LAPACK_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${LAPACK_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
BUILD_BYPRODUCTS
|
||||
${LAPACK_STATIC_LIB})
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}")
|
||||
add_library(lapack STATIC IMPORTED)
|
||||
set_target_properties(
|
||||
lapack
|
||||
PROPERTIES IMPORTED_LOCATION "${LAPACK_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${LAPACK_INCLUDE_DIR}")
|
||||
|
||||
add_dependencies(lapack lapack_ep)
|
||||
endmacro()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# FAISS
|
||||
|
||||
if(NOT DEFINED BUILD_FAISS_WITH_MKL)
|
||||
set(BUILD_FAISS_WITH_MKL OFF)
|
||||
endif()
|
||||
|
||||
if(EXISTS "/proc/cpuinfo")
|
||||
FILE(READ /proc/cpuinfo PROC_CPUINFO)
|
||||
|
||||
SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n")
|
||||
STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}")
|
||||
STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}")
|
||||
|
||||
if(NOT ${VENDOR_ID} STREQUAL "GenuineIntel")
|
||||
set(BUILD_FAISS_WITH_MKL OFF)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
macro(build_faiss)
|
||||
message(STATUS "Building FAISS-${FAISS_VERSION} from source")
|
||||
set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep")
|
||||
set(FAISS_INCLUDE_DIR "${FAISS_PREFIX}/include")
|
||||
set(FAISS_STATIC_LIB
|
||||
"${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
|
||||
set(FAISS_CONFIGURE_ARGS
|
||||
"--prefix=${FAISS_PREFIX}"
|
||||
"CFLAGS=${EP_C_FLAGS}"
|
||||
"CXXFLAGS=${EP_CXX_FLAGS}"
|
||||
--without-python)
|
||||
|
||||
set(FAISS_CFLAGS ${EP_C_FLAGS})
|
||||
set(FAISS_CXXFLAGS ${EP_CXX_FLAGS})
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
message(STATUS "Build Faiss with MKL")
|
||||
if(NOT DEFINED MKL_LIB_PATH)
|
||||
set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64")
|
||||
message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}")
|
||||
endif()
|
||||
|
||||
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
|
||||
"CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include"
|
||||
"LDFLAGS=-L${MKL_LIB_PATH}"
|
||||
"LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl")
|
||||
|
||||
else()
|
||||
message(STATUS "Build Faiss with OpenBlas/LAPACK")
|
||||
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
|
||||
"LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib")
|
||||
endif()
|
||||
|
||||
if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
|
||||
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
|
||||
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
|
||||
"--with-cuda-arch=-gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61 -gencode=arch=compute_75,code=sm_75"
|
||||
)
|
||||
else()
|
||||
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
|
||||
endif()
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}")
|
||||
set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz")
|
||||
set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}")
|
||||
set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${FAISS_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote file ${FAISS_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(faiss_ep
|
||||
URL
|
||||
${FAISS_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
"./configure"
|
||||
${FAISS_CONFIGURE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
INSTALL_COMMAND
|
||||
${MAKE} install
|
||||
BUILD_BYPRODUCTS
|
||||
${FAISS_STATIC_LIB})
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
|
||||
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
|
||||
endif()
|
||||
|
||||
ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(faiss_ep
|
||||
URL
|
||||
${FAISS_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
"./configure"
|
||||
${FAISS_CONFIGURE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
INSTALL_COMMAND
|
||||
${MAKE} install
|
||||
BUILD_BYPRODUCTS
|
||||
${FAISS_STATIC_LIB})
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
|
||||
ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}")
|
||||
add_library(faiss SHARED IMPORTED)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")
|
||||
set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a
|
||||
${MKL_LIB_PATH}/libmkl_gnu_thread.a
|
||||
${MKL_LIB_PATH}/libmkl_core.a)
|
||||
|
||||
set_target_properties(
|
||||
faiss
|
||||
PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" )
|
||||
else()
|
||||
set_target_properties(
|
||||
faiss
|
||||
PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "openblas;lapack" )
|
||||
endif()
|
||||
|
||||
add_dependencies(faiss faiss_ep)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
|
||||
add_dependencies(faiss openblas_ep)
|
||||
add_dependencies(faiss lapack_ep)
|
||||
endif()
|
||||
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_FAISS)
|
||||
|
||||
if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF")
|
||||
resolve_dependency(OpenBLAS)
|
||||
get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}")
|
||||
link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib)
|
||||
|
||||
resolve_dependency(LAPACK)
|
||||
get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}")
|
||||
link_directories(SYSTEM "${LAPACK_PREFIX}/lib")
|
||||
endif()
|
||||
|
||||
resolve_dependency(FAISS)
|
||||
get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM "${FAISS_INCLUDE_DIR}")
|
||||
link_directories(SYSTEM ${FAISS_PREFIX}/lib/)
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Google gtest
|
||||
|
||||
|
|
@ -1232,30 +674,6 @@ if (MILVUS_BUILD_TESTS)
|
|||
include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# JSONCONS
|
||||
|
||||
macro(build_jsoncons)
|
||||
message(STATUS "Building JSONCONS-${JSONCONS_VERSION} from source")
|
||||
|
||||
set(JSONCONS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jsoncons_ep-prefix")
|
||||
set(JSONCONS_TAR_NAME "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}.tar.gz")
|
||||
set(JSONCONS_INCLUDE_DIR "${JSONCONS_PREFIX}/jsoncons-${JSONCONS_VERSION}/include")
|
||||
if (NOT EXISTS ${JSONCONS_INCLUDE_DIR})
|
||||
file(MAKE_DIRECTORY ${JSONCONS_PREFIX})
|
||||
file(DOWNLOAD ${JSONCONS_SOURCE_URL}
|
||||
${JSONCONS_TAR_NAME})
|
||||
execute_process(COMMAND ${CMAKE_COMMAND} -E tar -xf ${JSONCONS_TAR_NAME}
|
||||
WORKING_DIRECTORY ${JSONCONS_PREFIX})
|
||||
|
||||
endif ()
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_JSONCONS)
|
||||
resolve_dependency(JSONCONS)
|
||||
include_directories(SYSTEM "${JSONCONS_INCLUDE_DIR}")
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# lz4
|
||||
|
||||
|
|
@ -1563,95 +981,6 @@ if(MILVUS_WITH_PROMETHEUS)
|
|||
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# RocksDB
|
||||
|
||||
macro(build_rocksdb)
|
||||
message(STATUS "Building RocksDB-${ROCKSDB_VERSION} from source")
|
||||
set(ROCKSDB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix/src/rocksdb_ep")
|
||||
set(ROCKSDB_INCLUDE_DIRS "${ROCKSDB_PREFIX}/include")
|
||||
set(ROCKSDB_STATIC_LIB_NAME rocksdb)
|
||||
set(ROCKSDB_STATIC_LIB
|
||||
"${ROCKSDB_PREFIX}/lib/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ROCKSDB_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
|
||||
)
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
string(MD5 ROCKSDB_COMBINE_MD5 "${ROCKSDB_LAST_COMMIT_ID}")
|
||||
set(ROCKSDB_CACHE_PACKAGE_NAME "rocksdb_${ROCKSDB_MD5}.tar.gz")
|
||||
set(ROCKSDB_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ROCKSDB_CACHE_PACKAGE_NAME}")
|
||||
set(ROCKSDB_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ROCKSDB_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${ROCKSDB_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote file ${ROCKSDB_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(rocksdb_ep
|
||||
URL
|
||||
${ROCKSDB_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
""
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
static_lib
|
||||
"prefix=${ROCKSDB_PREFIX}"
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
INSTALL_COMMAND
|
||||
${MAKE}
|
||||
install-static
|
||||
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
|
||||
BUILD_BYPRODUCTS
|
||||
"${ROCKSDB_STATIC_LIB}")
|
||||
|
||||
ExternalProject_Create_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/rocksdb_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ROCKSDB_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${ROCKSDB_CACHE_URL} ${ROCKSDB_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${ROCKSDB_CACHE_URL} TO ${ROCKSDB_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(rocksdb_ep ${ROCKSDB_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(rocksdb_ep
|
||||
URL
|
||||
${ROCKSDB_SOURCE_URL}
|
||||
${EP_LOG_OPTIONS}
|
||||
CONFIGURE_COMMAND
|
||||
""
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
static_lib
|
||||
"prefix=${ROCKSDB_PREFIX}"
|
||||
BUILD_IN_SOURCE
|
||||
1
|
||||
INSTALL_COMMAND
|
||||
${MAKE}
|
||||
install-static
|
||||
"INSTALL_PATH=${ROCKSDB_PREFIX}/lib"
|
||||
BUILD_BYPRODUCTS
|
||||
"${ROCKSDB_STATIC_LIB}")
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${ROCKSDB_PREFIX}/include")
|
||||
|
||||
add_library(rocksdb STATIC IMPORTED)
|
||||
set_target_properties(rocksdb
|
||||
PROPERTIES IMPORTED_LOCATION "${ROCKSDB_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${ROCKSDB_INCLUDE_DIRS}")
|
||||
add_dependencies(rocksdb rocksdb_ep)
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_ROCKSDB)
|
||||
|
||||
resolve_dependency(RocksDB)
|
||||
|
||||
link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/)
|
||||
include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# Snappy
|
||||
|
||||
|
|
@ -2091,116 +1420,6 @@ if(MILVUS_WITH_ZSTD)
|
|||
include_directories(SYSTEM ${ZSTD_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# aws
|
||||
macro(build_aws)
|
||||
message(STATUS "Building aws-${AWS_VERSION} from source")
|
||||
set(AWS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix/src/aws_ep")
|
||||
|
||||
set(AWS_CMAKE_ARGS
|
||||
${EP_COMMON_TOOLCHAIN}
|
||||
"-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}"
|
||||
-DCMAKE_BUILD_TYPE=Release
|
||||
-DCMAKE_INSTALL_LIBDIR=lib
|
||||
-DBUILD_ONLY=s3
|
||||
-DBUILD_SHARED_LIBS=off
|
||||
-DENABLE_TESTING=off
|
||||
-DENABLE_UNITY_BUILD=on
|
||||
-DNO_ENCRYPTION=off)
|
||||
|
||||
set(AWS_CPP_SDK_CORE_STATIC_LIB
|
||||
"${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
set(AWS_CPP_SDK_S3_STATIC_LIB
|
||||
"${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}")
|
||||
set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include")
|
||||
set(AWS_CMAKE_ARGS
|
||||
${AWS_CMAKE_ARGS}
|
||||
-DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
|
||||
-DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
|
||||
-DCMAKE_C_FLAGS=${EP_C_FLAGS}
|
||||
-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS})
|
||||
|
||||
if(USE_JFROG_CACHE STREQUAL "ON")
|
||||
set(AWS_CACHE_PACKAGE_NAME "aws_${AWS_MD5}.tar.gz")
|
||||
set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}")
|
||||
set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}")
|
||||
|
||||
execute_process(COMMAND wget -q --method HEAD ${AWS_CACHE_URL} RESULT_VARIABLE return_code)
|
||||
message(STATUS "Check the remote file ${AWS_CACHE_URL}. return code = ${return_code}")
|
||||
if (NOT return_code EQUAL 0)
|
||||
externalproject_add(aws_ep
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${AWS_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_DIR
|
||||
${AWS_PREFIX}
|
||||
URL
|
||||
${AWS_SOURCE_URL}
|
||||
BUILD_BYPRODUCTS
|
||||
"${AWS_CPP_SDK_S3_STATIC_LIB}"
|
||||
"${AWS_CPP_SDK_CORE_STATIC_LIB}")
|
||||
|
||||
ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL})
|
||||
else()
|
||||
file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status)
|
||||
list(GET status 0 status_code)
|
||||
message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
|
||||
if (status_code EQUAL 0)
|
||||
ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
externalproject_add(aws_ep
|
||||
${EP_LOG_OPTIONS}
|
||||
CMAKE_ARGS
|
||||
${AWS_CMAKE_ARGS}
|
||||
BUILD_COMMAND
|
||||
${MAKE}
|
||||
${MAKE_BUILD_ARGS}
|
||||
INSTALL_DIR
|
||||
${AWS_PREFIX}
|
||||
URL
|
||||
${AWS_SOURCE_URL}
|
||||
BUILD_BYPRODUCTS
|
||||
"${AWS_CPP_SDK_S3_STATIC_LIB}"
|
||||
"${AWS_CPP_SDK_CORE_STATIC_LIB}")
|
||||
endif()
|
||||
|
||||
file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}")
|
||||
add_library(aws-cpp-sdk-s3 STATIC IMPORTED)
|
||||
set_target_properties(aws-cpp-sdk-s3
|
||||
PROPERTIES
|
||||
IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a")
|
||||
|
||||
add_library(aws-cpp-sdk-core STATIC IMPORTED)
|
||||
set_target_properties(aws-cpp-sdk-core
|
||||
PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}"
|
||||
INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}"
|
||||
INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a")
|
||||
|
||||
add_dependencies(aws-cpp-sdk-s3 aws_ep)
|
||||
add_dependencies(aws-cpp-sdk-core aws_ep)
|
||||
|
||||
endmacro()
|
||||
|
||||
if(MILVUS_WITH_AWS)
|
||||
resolve_dependency(AWS)
|
||||
|
||||
link_directories(SYSTEM ${AWS_PREFIX}/lib)
|
||||
|
||||
get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM ${AWS_CPP_SDK_S3_INCLUDE_DIR})
|
||||
|
||||
get_target_property(AWS_CPP_SDK_CORE_INCLUDE_DIR aws-cpp-sdk-core INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM ${AWS_CPP_SDK_CORE_INCLUDE_DIR})
|
||||
|
||||
endif()
|
||||
|
||||
# ----------------------------------------------------------------------
|
||||
# libunwind
|
||||
|
||||
|
|
@ -2465,12 +1684,14 @@ macro(build_grpc)
|
|||
add_dependencies(grpc_protoc grpc_ep)
|
||||
endmacro()
|
||||
|
||||
resolve_dependency(GRPC)
|
||||
if(MILVUS_WITH_GRPC)
|
||||
resolve_dependency(GRPC)
|
||||
|
||||
get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
|
||||
link_directories(SYSTEM ${GRPC_PREFIX}/lib)
|
||||
get_target_property(GRPC_INCLUDE_DIR grpc INTERFACE_INCLUDE_DIRECTORIES)
|
||||
include_directories(SYSTEM ${GRPC_INCLUDE_DIR})
|
||||
link_directories(SYSTEM ${GRPC_PREFIX}/lib)
|
||||
|
||||
set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
|
||||
include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
|
||||
link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
|
||||
set(GRPC_THIRD_PARTY_DIR ${CMAKE_CURRENT_BINARY_DIR}/grpc_ep-prefix/src/grpc_ep/third_party)
|
||||
include_directories(SYSTEM ${GRPC_THIRD_PARTY_DIR}/protobuf/src)
|
||||
link_directories(SYSTEM ${GRPC_PROTOBUF_LIB_DIR})
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -104,6 +104,8 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \
|
|||
"src/server/Server.cpp"\
|
||||
"src/server/DBWrapper.cpp"\
|
||||
"src/server/grpc_impl/GrpcMilvusServer.cpp"\
|
||||
"src/utils/easylogging++.h"\
|
||||
"src/utils/easylogging++.cc"\
|
||||
|
||||
# gen html report
|
||||
${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/
|
||||
|
|
@ -17,69 +17,65 @@
|
|||
# under the License.
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
include_directories(${MILVUS_SOURCE_DIR})
|
||||
include_directories(${MILVUS_ENGINE_SRC})
|
||||
|
||||
add_subdirectory(core)
|
||||
|
||||
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
foreach(dir ${CORE_INCLUDE_DIRS})
|
||||
foreach (dir ${CORE_INCLUDE_DIRS})
|
||||
include_directories(${dir})
|
||||
endforeach()
|
||||
endforeach ()
|
||||
|
||||
aux_source_directory(cache cache_files)
|
||||
aux_source_directory(config config_files)
|
||||
aux_source_directory(server server_files)
|
||||
aux_source_directory(server/grpc_impl grpcserver_files)
|
||||
aux_source_directory(utils utils_files)
|
||||
aux_source_directory(db db_main_files)
|
||||
aux_source_directory(db/engine db_engine_files)
|
||||
aux_source_directory(db/insert db_insert_files)
|
||||
aux_source_directory(db/meta db_meta_files)
|
||||
aux_source_directory(metrics metrics_files)
|
||||
aux_source_directory(wrapper/knowhere knowhere_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files)
|
||||
|
||||
aux_source_directory(scheduler/action scheduler_action_files)
|
||||
aux_source_directory(scheduler/event scheduler_event_files)
|
||||
aux_source_directory(scheduler/job scheduler_job_files)
|
||||
aux_source_directory(scheduler/resource scheduler_resource_files)
|
||||
aux_source_directory(scheduler/task scheduler_task_files)
|
||||
aux_source_directory(scheduler scheduler_root_files)
|
||||
set(scheduler_srcs
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
|
||||
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/insert db_insert_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/meta db_meta_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler db_scheduler_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context db_scheduler_context_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task db_scheduler_task_files)
|
||||
set(db_scheduler_files
|
||||
${db_scheduler_files}
|
||||
${db_scheduler_context_files}
|
||||
${db_scheduler_task_files}
|
||||
)
|
||||
|
||||
set(grpc_service_files
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
|
||||
)
|
||||
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files)
|
||||
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler scheduler_main_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files)
|
||||
set(scheduler_files
|
||||
${scheduler_main_files}
|
||||
${scheduler_action_files}
|
||||
${scheduler_event_files}
|
||||
${scheduler_job_files}
|
||||
${scheduler_resource_files}
|
||||
${scheduler_task_files}
|
||||
${scheduler_root_files}
|
||||
)
|
||||
|
||||
aux_source_directory(db/scheduler scheduler_files)
|
||||
aux_source_directory(db/scheduler/context scheduler_context_files)
|
||||
aux_source_directory(db/scheduler/task scheduler_task_files)
|
||||
set(db_scheduler_files
|
||||
${scheduler_files}
|
||||
${scheduler_context_files}
|
||||
${scheduler_task_files}
|
||||
)
|
||||
set(license_check_files
|
||||
license/LicenseLibrary.cpp
|
||||
license/LicenseCheck.cpp
|
||||
)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/server server_files)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/server/grpc_impl grpc_server_files)
|
||||
|
||||
set(license_generator_files
|
||||
license/LicenseGenerator.cpp
|
||||
license/LicenseLibrary.cpp
|
||||
)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/utils utils_files)
|
||||
|
||||
set(grpc_service_files
|
||||
metrics/SystemInfo.cpp
|
||||
metrics/SystemInfo.h
|
||||
grpc/gen-milvus/milvus.grpc.pb.cc
|
||||
grpc/gen-milvus/milvus.pb.cc
|
||||
grpc/gen-status/status.grpc.pb.cc
|
||||
grpc/gen-status/status.pb.cc
|
||||
scheduler/Utils.h)
|
||||
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_files)
|
||||
|
||||
set(db_files
|
||||
set(engine_files
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
|
||||
${cache_files}
|
||||
${db_main_files}
|
||||
|
|
@ -88,38 +84,40 @@ set(db_files
|
|||
${db_meta_files}
|
||||
${db_scheduler_files}
|
||||
${metrics_files}
|
||||
${knowhere_files}
|
||||
${utils_files}
|
||||
${wrapper_files}
|
||||
)
|
||||
|
||||
set(s3_client_files
|
||||
storage/s3/S3ClientWrapper.cpp
|
||||
storage/s3/S3ClientWrapper.h)
|
||||
|
||||
include_directories(/usr/include)
|
||||
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
|
||||
include_directories(/usr/include/mysql)
|
||||
include_directories(grpc/gen-status)
|
||||
include_directories(grpc/gen-milvus)
|
||||
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-status)
|
||||
include_directories(${MILVUS_ENGINE_SRC}/grpc/gen-milvus)
|
||||
|
||||
set(client_grpc_lib
|
||||
grpcpp_channelz
|
||||
grpc++
|
||||
grpc
|
||||
grpc_protobuf
|
||||
grpc_protoc)
|
||||
grpc_protoc
|
||||
)
|
||||
|
||||
set(third_party_libs
|
||||
knowhere
|
||||
sqlite
|
||||
${client_grpc_lib}
|
||||
yaml-cpp
|
||||
set(prometheus_lib
|
||||
prometheus-cpp-push
|
||||
prometheus-cpp-pull
|
||||
prometheus-cpp-core
|
||||
)
|
||||
|
||||
set(boost_lib
|
||||
boost_system_static
|
||||
boost_filesystem_static
|
||||
boost_serialization_static
|
||||
)
|
||||
|
||||
set(third_party_libs
|
||||
sqlite
|
||||
${client_grpc_lib}
|
||||
yaml-cpp
|
||||
${prometheus_lib}
|
||||
${boost_lib}
|
||||
bzip2
|
||||
lz4
|
||||
snappy
|
||||
|
|
@ -131,13 +129,12 @@ set(third_party_libs
|
|||
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
|
||||
cudart
|
||||
)
|
||||
|
||||
if (MILVUS_ENABLE_PROFILING STREQUAL "ON")
|
||||
set(third_party_libs ${third_party_libs}
|
||||
gperftools
|
||||
libunwind)
|
||||
endif()
|
||||
|
||||
libunwind
|
||||
)
|
||||
endif ()
|
||||
|
||||
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
|
||||
set(engine_libs
|
||||
|
|
@ -149,22 +146,19 @@ set(engine_libs
|
|||
|
||||
if (NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
|
||||
set(engine_libs
|
||||
${engine_libs}
|
||||
libquadmath.a
|
||||
)
|
||||
${engine_libs}
|
||||
libquadmath.a
|
||||
)
|
||||
endif ()
|
||||
|
||||
|
||||
cuda_add_library(milvus_engine STATIC ${db_files})
|
||||
cuda_add_library(milvus_engine STATIC ${engine_files})
|
||||
target_link_libraries(milvus_engine ${engine_libs} ${third_party_libs})
|
||||
|
||||
add_library(metrics STATIC ${metrics_files})
|
||||
|
||||
set(metrics_lib
|
||||
yaml-cpp
|
||||
prometheus-cpp-push
|
||||
prometheus-cpp-pull
|
||||
prometheus-cpp-core
|
||||
${prometheus_lib}
|
||||
)
|
||||
|
||||
target_link_libraries(metrics ${metrics_lib})
|
||||
|
|
@ -176,21 +170,17 @@ set(server_libs
|
|||
metrics
|
||||
)
|
||||
|
||||
set(knowhere_libs
|
||||
knowhere
|
||||
)
|
||||
|
||||
add_executable(milvus_server
|
||||
${config_files}
|
||||
${server_files}
|
||||
${grpcserver_files}
|
||||
${utils_files}
|
||||
${grpc_service_files}
|
||||
${metrics_files}
|
||||
${scheduler_srcs}
|
||||
${scheduler_files}
|
||||
${server_files}
|
||||
${grpc_server_files}
|
||||
${grpc_service_files}
|
||||
${utils_files}
|
||||
)
|
||||
|
||||
target_link_libraries(milvus_server ${server_libs} ${knowhere_libs} ${third_party_libs})
|
||||
target_link_libraries(milvus_server ${server_libs} knowhere ${third_party_libs})
|
||||
|
||||
install(TARGETS milvus_server DESTINATION bin)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "wrapper/knowhere/vec_index.h"
|
||||
#include "src/wrapper/vec_index.h"
|
||||
|
||||
#include <memory>
|
||||
|
||||
|
|
|
|||
|
|
@ -84,7 +84,7 @@ include(DefineOptionsCore)
|
|||
include(BuildUtilsCore)
|
||||
include(ThirdPartyPackagesCore)
|
||||
|
||||
add_subdirectory(src)
|
||||
add_subdirectory(knowhere)
|
||||
|
||||
if (BUILD_COVERAGE STREQUAL "ON")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage")
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <knowhere/common/array.h>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
ArrayPtr
|
||||
CopyArray(const ArrayPtr &origin);
|
||||
|
||||
SchemaPtr
|
||||
CopySchema(const SchemaPtr &origin);
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <knowhere/common/dataset.h>
|
||||
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
std::shared_ptr<SPTAG::VectorSet>
|
||||
ConvertToVectorSet(const DatasetPtr &dataset);
|
||||
|
||||
std::shared_ptr<SPTAG::MetadataSet>
|
||||
ConvertToMetadataSet(const DatasetPtr &dataset);
|
||||
|
||||
std::vector<SPTAG::QueryResult>
|
||||
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
|
||||
|
||||
DatasetPtr
|
||||
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <jsoncons/json.hpp>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using Config = jsoncons::json;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace sched {
|
||||
namespace master {
|
||||
|
||||
} // namespace master
|
||||
} // namespace sched
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include "zlibrary/error/error.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
using Error = zilliz::lib::ErrorCode;
|
||||
|
||||
constexpr Error STORE_SUCCESS = zilliz::lib::SUCCESS_CODE;
|
||||
|
||||
constexpr Error ERROR_CODE_BASE = 0x36000;
|
||||
constexpr Error ERROR_CODE_END = 0x37000;
|
||||
|
||||
constexpr Error
|
||||
ToGlobalErrorCode(const Error error_code) {
|
||||
return zilliz::lib::ToGlobalErrorCode(error_code, ERROR_CODE_BASE);
|
||||
}
|
||||
|
||||
class Exception : public zilliz::lib::Exception {
|
||||
public:
|
||||
Exception(const Error error_code,
|
||||
const std::string &message = nullptr)
|
||||
: zilliz::lib::Exception(error_code, "KNOWHERE", message) {}
|
||||
};
|
||||
|
||||
constexpr Error UNEXPECTED = ToGlobalErrorCode(0x001);
|
||||
constexpr Error UNSUPPORTED = ToGlobalErrorCode(0x002);
|
||||
constexpr Error NULL_POINTER = ToGlobalErrorCode(0x003);
|
||||
constexpr Error OVERFLOW = ToGlobalErrorCode(0x004);
|
||||
constexpr Error INVALID_ARGUMENT = ToGlobalErrorCode(0x005);
|
||||
constexpr Error UNSUPPORTED_TYPE = ToGlobalErrorCode(0x006);
|
||||
|
||||
|
||||
} // namespace store
|
||||
} // namespace zilliz
|
||||
|
||||
using Error = zilliz::store::Error;
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
//#include "zcommon/id/id.h"
|
||||
//using ID = zilliz::common::ID;
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
|
||||
class ID {
|
||||
public:
|
||||
constexpr static int64_t kIDSize = 20;
|
||||
|
||||
public:
|
||||
const int32_t *
|
||||
data() const { return content_; }
|
||||
|
||||
int32_t *
|
||||
mutable_data() { return content_; }
|
||||
|
||||
bool
|
||||
IsValid() const;
|
||||
|
||||
std::string
|
||||
ToString() const;
|
||||
|
||||
bool
|
||||
operator==(const ID &that) const;
|
||||
|
||||
bool
|
||||
operator<(const ID &that) const;
|
||||
|
||||
protected:
|
||||
int32_t content_[5] = {};
|
||||
};
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "arrow/type.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using DataType = arrow::DataType;
|
||||
using Field = arrow::Field;
|
||||
using FieldPtr = std::shared_ptr<arrow::Field>;
|
||||
using Schema = arrow::Schema;
|
||||
using SchemaPtr = std::shared_ptr<Schema>;
|
||||
using SchemaConstPtr = std::shared_ptr<const Schema>;
|
||||
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "arrow/tensor.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using Tensor = arrow::Tensor;
|
||||
using TensorPtr = std::shared_ptr<Tensor>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "knowhere/common/binary_set.h"
|
||||
#include "knowhere/common/dataset.h"
|
||||
#include "knowhere/index/index_type.h"
|
||||
#include "knowhere/index/index_model.h"
|
||||
#include "knowhere/index/preprocessor/preprocessor.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class Index {
|
||||
public:
|
||||
virtual BinarySet
|
||||
Serialize() = 0;
|
||||
|
||||
virtual void
|
||||
Load(const BinarySet &index_binary) = 0;
|
||||
|
||||
// @throw
|
||||
virtual DatasetPtr
|
||||
Search(const DatasetPtr &dataset, const Config &config) = 0;
|
||||
|
||||
public:
|
||||
IndexType
|
||||
idx_type() const { return idx_type_; }
|
||||
|
||||
void
|
||||
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
|
||||
|
||||
virtual void
|
||||
set_preprocessor(PreprocessorPtr preprocessor) {}
|
||||
|
||||
virtual void
|
||||
set_index_model(IndexModelPtr model) {}
|
||||
|
||||
private:
|
||||
IndexType idx_type_;
|
||||
};
|
||||
|
||||
|
||||
using IndexPtr = std::shared_ptr<Index>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "knowhere/common/binary_set.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class IndexModel {
|
||||
public:
|
||||
virtual BinarySet
|
||||
Serialize() = 0;
|
||||
|
||||
virtual void
|
||||
Load(const BinarySet &binary) = 0;
|
||||
};
|
||||
|
||||
using IndexModelPtr = std::shared_ptr<IndexModel>;
|
||||
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
enum class IndexType {
|
||||
kUnknown = 0,
|
||||
kVecIdxBegin = 100,
|
||||
kVecIVFFlat = kVecIdxBegin,
|
||||
kVecIdxEnd,
|
||||
};
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "knowhere/common/dataset.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class Preprocessor {
|
||||
public:
|
||||
virtual DatasetPtr
|
||||
Preprocess(const DatasetPtr &input) = 0;
|
||||
};
|
||||
|
||||
|
||||
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
#define META_ROWS ("rows")
|
||||
#define META_DIM ("dimension")
|
||||
#define META_K ("k")
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,34 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
using KDTParameter = std::pair<std::string, std::string>;
|
||||
|
||||
class KDTParameterManagement {
|
||||
public:
|
||||
const std::vector<KDTParameter> &
|
||||
GetKDTParameters();
|
||||
|
||||
public:
|
||||
static KDTParameterManagement &
|
||||
GetInstance() {
|
||||
static KDTParameterManagement instance;
|
||||
return instance;
|
||||
}
|
||||
|
||||
KDTParameterManagement(const KDTParameterManagement &) = delete;
|
||||
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
|
||||
private:
|
||||
KDTParameterManagement();
|
||||
|
||||
private:
|
||||
std::vector<KDTParameter> kdt_parameters_;
|
||||
};
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
%module nsg
|
||||
%{
|
||||
#define SWIG_FILE_WITH_INIT
|
||||
#include <numpy/arrayobject.h>
|
||||
|
||||
/* Include the header in the wrapper code */
|
||||
#include "nsg.h"
|
||||
|
||||
|
||||
%}
|
||||
|
||||
|
||||
/* Parse the header file */
|
||||
%include "index.h"
|
||||
|
||||
|
|
@ -5,7 +5,7 @@ include_directories(${TBB_DIR}/include)
|
|||
include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
|
||||
link_directories(${CUDA_TOOLKIT_ROOT_DIR}/lib64)
|
||||
|
||||
include_directories(${CORE_SOURCE_DIR}/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/knowhere)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
|
|
@ -107,7 +107,7 @@ INSTALL(FILES ${CORE_SOURCE_DIR}/thirdparty/tbb/libtbb.so
|
|||
)
|
||||
|
||||
set(CORE_INCLUDE_DIRS
|
||||
${CORE_SOURCE_DIR}/include
|
||||
${CORE_SOURCE_DIR}/knowhere
|
||||
${CORE_SOURCE_DIR}/thirdparty
|
||||
${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include
|
||||
|
|
@ -120,18 +120,18 @@ set(CORE_INCLUDE_DIRS
|
|||
|
||||
set(CORE_INCLUDE_DIRS ${CORE_INCLUDE_DIRS} PARENT_SCOPE)
|
||||
|
||||
INSTALL(DIRECTORY
|
||||
${CORE_SOURCE_DIR}/include/knowhere
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
|
||||
${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
|
||||
${ARROW_INCLUDE_DIR}/arrow
|
||||
${FAISS_PREFIX}/include/faiss
|
||||
${OPENBLAS_INCLUDE_DIR}/
|
||||
${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
|
||||
DESTINATION
|
||||
include)
|
||||
|
||||
INSTALL(DIRECTORY
|
||||
${SPTAG_SOURCE_DIR}/AnnService/inc/
|
||||
DESTINATION
|
||||
include/SPTAG/AnnService/inc)
|
||||
#INSTALL(DIRECTORY
|
||||
# ${CORE_SOURCE_DIR}/include/knowhere
|
||||
# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons
|
||||
# ${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include/jsoncons_ext
|
||||
# ${ARROW_INCLUDE_DIR}/arrow
|
||||
# ${FAISS_PREFIX}/include/faiss
|
||||
# ${OPENBLAS_INCLUDE_DIR}/
|
||||
# ${CORE_SOURCE_DIR}/thirdparty/tbb/include/tbb
|
||||
# DESTINATION
|
||||
# include)
|
||||
#
|
||||
#INSTALL(DIRECTORY
|
||||
# ${SPTAG_SOURCE_DIR}/AnnService/inc/
|
||||
# DESTINATION
|
||||
# include/SPTAG/AnnService/inc)
|
||||
|
|
@ -1,5 +1,22 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "knowhere/adapter/arrow.h"
|
||||
|
||||
#include "arrow.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "knowhere/common/array.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
ArrayPtr
|
||||
CopyArray(const ArrayPtr &origin);
|
||||
|
||||
SchemaPtr
|
||||
CopySchema(const SchemaPtr &origin);
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,7 +1,23 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
#include "knowhere/index/vector_index/definitions.h"
|
||||
#include "knowhere/adapter/sptag.h"
|
||||
#include "knowhere/adapter/structure.h"
|
||||
#include "sptag.h"
|
||||
#include "structure.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
|
||||
|
||||
#include "knowhere/common/dataset.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
std::shared_ptr<SPTAG::VectorSet>
|
||||
ConvertToVectorSet(const DatasetPtr &dataset);
|
||||
|
||||
std::shared_ptr<SPTAG::MetadataSet>
|
||||
ConvertToMetadataSet(const DatasetPtr &dataset);
|
||||
|
||||
std::vector<SPTAG::QueryResult>
|
||||
ConvertToQueryResult(const DatasetPtr &dataset, const Config &config);
|
||||
|
||||
DatasetPtr
|
||||
ConvertToDataset(std::vector<SPTAG::QueryResult> query_results);
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -16,7 +16,7 @@
|
|||
// under the License.
|
||||
|
||||
|
||||
#include "knowhere/adapter/structure.h"
|
||||
#include "structure.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <knowhere/common/dataset.h>
|
||||
#include "knowhere/common/dataset.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,7 +1,25 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "arrow/array.h"
|
||||
#include "knowhere/common/schema.h"
|
||||
#include "schema.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,3 +1,21 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
|
|
@ -5,7 +23,7 @@
|
|||
#include <vector>
|
||||
#include <memory>
|
||||
|
||||
#include "knowhere/common/id.h"
|
||||
#include "id.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,6 +1,25 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "arrow/buffer.h"
|
||||
|
||||
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jsoncons/json.hpp>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using Config = jsoncons::json;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -1,12 +1,31 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <memory>
|
||||
#include "knowhere/common/array.h"
|
||||
#include "knowhere/common/buffer.h"
|
||||
#include "knowhere/common/tensor.h"
|
||||
#include "knowhere/common/schema.h"
|
||||
#include "knowhere/common/config.h"
|
||||
|
||||
#include "array.h"
|
||||
#include "buffer.h"
|
||||
#include "tensor.h"
|
||||
#include "schema.h"
|
||||
#include "config.h"
|
||||
#include "knowhere/adapter/arrow.h"
|
||||
|
||||
|
||||
|
|
@ -16,9 +16,10 @@
|
|||
// under the License.
|
||||
|
||||
|
||||
#include "knowhere/common/exception.h"
|
||||
#include <cstdio>
|
||||
|
||||
#include "exception.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
|
@ -18,33 +18,43 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "db/Status.h"
|
||||
//#include "zcommon/id/id.h"
|
||||
//using ID = zilliz::common::ID;
|
||||
|
||||
#include <stdint.h>
|
||||
#include <string>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace milvus {
|
||||
namespace engine {
|
||||
namespace storage {
|
||||
namespace knowhere {
|
||||
|
||||
class IStorage {
|
||||
|
||||
|
||||
class ID {
|
||||
public:
|
||||
virtual Status Create(const std::string &ip_address,
|
||||
const std::string &port,
|
||||
const std::string &access_key,
|
||||
const std::string &secret_key) = 0;
|
||||
constexpr static int64_t kIDSize = 20;
|
||||
|
||||
virtual Status Close() = 0;
|
||||
public:
|
||||
const int32_t *
|
||||
data() const { return content_; }
|
||||
|
||||
virtual Status CreateBucket(std::string& bucket_name) = 0;
|
||||
virtual Status DeleteBucket(std::string& bucket_name) = 0;
|
||||
virtual Status UploadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
|
||||
virtual Status DownloadFile(std::string &bucket_name, std::string &object_key, std::string &path_key) = 0;
|
||||
virtual Status DeleteFile(std::string &bucket_name, std::string &object_key) = 0;
|
||||
int32_t *
|
||||
mutable_data() { return content_; }
|
||||
|
||||
bool
|
||||
IsValid() const;
|
||||
|
||||
std::string
|
||||
ToString() const;
|
||||
|
||||
bool
|
||||
operator==(const ID &that) const;
|
||||
|
||||
bool
|
||||
operator<(const ID &that) const;
|
||||
|
||||
protected:
|
||||
int32_t content_[5] = {};
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "arrow/type.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using DataType = arrow::DataType;
|
||||
using Field = arrow::Field;
|
||||
using FieldPtr = std::shared_ptr<arrow::Field>;
|
||||
using Schema = arrow::Schema;
|
||||
using SchemaPtr = std::shared_ptr<Schema>;
|
||||
using SchemaConstPtr = std::shared_ptr<const Schema>;
|
||||
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "arrow/tensor.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
using Tensor = arrow::Tensor;
|
||||
using TensorPtr = std::shared_ptr<Tensor>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#include <iostream> // TODO(linxj): using Log instead
|
||||
|
||||
#include "knowhere/common/timer.h"
|
||||
#include "timer.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "knowhere/common/binary_set.h"
|
||||
#include "knowhere/common/dataset.h"
|
||||
#include "index_type.h"
|
||||
#include "index_model.h"
|
||||
#include "knowhere/index/preprocessor/preprocessor.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class Index {
|
||||
public:
|
||||
virtual BinarySet
|
||||
Serialize() = 0;
|
||||
|
||||
virtual void
|
||||
Load(const BinarySet &index_binary) = 0;
|
||||
|
||||
// @throw
|
||||
virtual DatasetPtr
|
||||
Search(const DatasetPtr &dataset, const Config &config) = 0;
|
||||
|
||||
public:
|
||||
IndexType
|
||||
idx_type() const { return idx_type_; }
|
||||
|
||||
void
|
||||
set_idx_type(IndexType idx_type) { idx_type_ = idx_type; }
|
||||
|
||||
virtual void
|
||||
set_preprocessor(PreprocessorPtr preprocessor) {}
|
||||
|
||||
virtual void
|
||||
set_index_model(IndexModelPtr model) {}
|
||||
|
||||
private:
|
||||
IndexType idx_type_;
|
||||
};
|
||||
|
||||
|
||||
using IndexPtr = std::shared_ptr<Index>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include "knowhere/common/binary_set.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class IndexModel {
|
||||
public:
|
||||
virtual BinarySet
|
||||
Serialize() = 0;
|
||||
|
||||
virtual void
|
||||
Load(const BinarySet &binary) = 0;
|
||||
};
|
||||
|
||||
using IndexModelPtr = std::shared_ptr<IndexModel>;
|
||||
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
enum class IndexType {
|
||||
kUnknown = 0,
|
||||
kVecIdxBegin = 100,
|
||||
kVecIVFFlat = kVecIdxBegin,
|
||||
kVecIdxEnd,
|
||||
};
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "knowhere/common/dataset.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
|
||||
class Preprocessor {
|
||||
public:
|
||||
virtual DatasetPtr
|
||||
Preprocess(const DatasetPtr &input) = 0;
|
||||
};
|
||||
|
||||
|
||||
using PreprocessorPtr = std::shared_ptr<Preprocessor>;
|
||||
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -17,10 +17,10 @@
|
|||
|
||||
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/index/vector_index/cloner.h"
|
||||
#include "knowhere/index/vector_index/ivf.h"
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
#include "knowhere/index/vector_index/idmap.h"
|
||||
#include "cloner.h"
|
||||
#include "ivf.h"
|
||||
#include "gpu_ivf.h"
|
||||
#include "idmap.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,3 +1,20 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#include <sstream>
|
||||
#include <SPTAG/AnnService/inc/Server/QueryParser.h>
|
||||
|
|
@ -7,10 +24,10 @@
|
|||
|
||||
#undef mkdir
|
||||
|
||||
#include "knowhere/index/vector_index/cpu_kdt_rng.h"
|
||||
#include "knowhere/index/vector_index/definitions.h"
|
||||
#include "cpu_kdt_rng.h"
|
||||
#include "definitions.h"
|
||||
//#include "knowhere/index/preprocessor/normalize.h"
|
||||
#include "knowhere/index/vector_index/kdt_parameters.h"
|
||||
#include "kdt_parameters.h"
|
||||
#include "knowhere/adapter/sptag.h"
|
||||
#include "knowhere/common/exception.h"
|
||||
|
||||
|
|
@ -1,8 +1,26 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include "knowhere/index/vector_index/vector_index.h"
|
||||
#include "vector_index.h"
|
||||
#include "knowhere/index/index_model.h"
|
||||
#include <SPTAG/AnnService/inc/Core/VectorIndex.h>
|
||||
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
#define META_ROWS ("rows")
|
||||
#define META_DIM ("dimension")
|
||||
#define META_K ("k")
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -27,9 +27,9 @@
|
|||
|
||||
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/index/vector_index/cloner.h"
|
||||
#include "cloner.h"
|
||||
#include "knowhere/adapter/faiss_adopt.h"
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
#include "gpu_ivf.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
|
|
@ -1,3 +1,21 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <faiss/gpu/StandardGpuResources.h>
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/adapter/faiss_adopt.h"
|
||||
#include "knowhere/index/vector_index/idmap.h"
|
||||
#include "idmap.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -25,13 +25,12 @@
|
|||
#include <faiss/index_io.h>
|
||||
#include <faiss/gpu/StandardGpuResources.h>
|
||||
#include <faiss/gpu/GpuAutoTune.h>
|
||||
#include <knowhere/index/vector_index/ivf.h>
|
||||
|
||||
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/index/vector_index/ivf.h"
|
||||
#include "ivf.h"
|
||||
#include "knowhere/adapter/faiss_adopt.h"
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
#include "gpu_ivf.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -25,7 +25,7 @@
|
|||
#include <faiss/AuxIndexStructures.h>
|
||||
#include <faiss/Index.h>
|
||||
|
||||
#include "knowhere/index/vector_index/vector_index.h"
|
||||
#include "vector_index.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,6 +1,24 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#include <mutex>
|
||||
#include "knowhere/index/vector_index/kdt_parameters.h"
|
||||
|
||||
#include "kdt_parameters.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -0,0 +1,52 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
||||
using KDTParameter = std::pair<std::string, std::string>;
|
||||
|
||||
class KDTParameterManagement {
|
||||
public:
|
||||
const std::vector<KDTParameter> &
|
||||
GetKDTParameters();
|
||||
|
||||
public:
|
||||
static KDTParameterManagement &
|
||||
GetInstance() {
|
||||
static KDTParameterManagement instance;
|
||||
return instance;
|
||||
}
|
||||
|
||||
KDTParameterManagement(const KDTParameterManagement &) = delete;
|
||||
KDTParameterManagement &operator=(const KDTParameterManagement &) = delete;
|
||||
private:
|
||||
KDTParameterManagement();
|
||||
|
||||
private:
|
||||
std::vector<KDTParameter> kdt_parameters_;
|
||||
};
|
||||
|
||||
} // namespace knowhere
|
||||
} // namespace zilliz
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "knowhere/index/vector_index/nsg/nsg.h"
|
||||
#include "nsg.h"
|
||||
|
||||
namespace zilliz {
|
||||
namespace knowhere {
|
||||
|
|
@ -22,7 +22,7 @@
|
|||
#include <stack>
|
||||
#include <omp.h>
|
||||
|
||||
#include "knowhere/index/vector_index/nsg/nsg.h"
|
||||
#include "nsg.h"
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/common/timer.h"
|
||||
#include "utils.h"
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#include <cstring>
|
||||
|
||||
#include "knowhere/index/vector_index/nsg/nsg_io.h"
|
||||
#include "nsg_io.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -23,7 +23,7 @@
|
|||
|
||||
#include <faiss/AutoTune.h>
|
||||
|
||||
#include "knowhere/index/vector_index/nsg/nsg.h"
|
||||
#include "nsg.h"
|
||||
#include "knowhere/common/config.h"
|
||||
|
||||
|
||||
|
|
@ -16,12 +16,12 @@
|
|||
// under the License.
|
||||
|
||||
|
||||
#include "knowhere/index/vector_index/nsg_index.h"
|
||||
#include "nsg_index.h"
|
||||
#include "knowhere/index/vector_index/nsg/nsg.h"
|
||||
#include "knowhere/index/vector_index/nsg/nsg_io.h"
|
||||
#include "knowhere/index/vector_index/idmap.h"
|
||||
#include "knowhere/index/vector_index/ivf.h"
|
||||
#include "knowhere/index/vector_index/gpu_ivf.h"
|
||||
#include "idmap.h"
|
||||
#include "ivf.h"
|
||||
#include "gpu_ivf.h"
|
||||
#include "knowhere/adapter/faiss_adopt.h"
|
||||
#include "knowhere/common/exception.h"
|
||||
#include "knowhere/common/timer.h"
|
||||
|
|
@ -18,7 +18,7 @@
|
|||
|
||||
#pragma once
|
||||
|
||||
#include "knowhere/index/vector_index/vector_index.h"
|
||||
#include "vector_index.h"
|
||||
|
||||
|
||||
namespace zilliz {
|
||||
|
|
@ -1,7 +1,26 @@
|
|||
// Licensed to the Apache Software Foundation (ASF) under one
|
||||
// or more contributor license agreements. See the NOTICE file
|
||||
// distributed with this work for additional information
|
||||
// regarding copyright ownership. The ASF licenses this file
|
||||
// to you under the Apache License, Version 2.0 (the
|
||||
// "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing,
|
||||
// software distributed under the License is distributed on an
|
||||
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
// KIND, either express or implied. See the License for the
|
||||
// specific language governing permissions and limitations
|
||||
// under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "knowhere/common/config.h"
|
||||
#include "knowhere/common/dataset.h"
|
||||
#include "knowhere/index/index.h"
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
include_directories(${CORE_SOURCE_DIR}/thirdparty)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/SPTAG/AnnService)
|
||||
include_directories(${CORE_SOURCE_DIR}/include)
|
||||
include_directories(${CORE_SOURCE_DIR}/knowhere)
|
||||
include_directories(${CORE_SOURCE_DIR}/thirdparty/jsoncons-0.126.0/include)
|
||||
include_directories(/usr/local/cuda/include)
|
||||
link_directories(/usr/local/cuda/lib64)
|
||||
|
|
@ -25,13 +25,13 @@ set(basic_libs
|
|||
|
||||
#<IVF-TEST>
|
||||
set(ivf_srcs
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
if(NOT TARGET test_ivf)
|
||||
|
|
@ -41,13 +41,13 @@ target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
|
|||
|
||||
#<IDMAP-TEST>
|
||||
set(idmap_srcs
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/idmap.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/ivf.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cloner.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/gpu_ivf.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
if(NOT TARGET test_idmap)
|
||||
|
|
@ -57,14 +57,14 @@ target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
|
|||
|
||||
#<KDT-TEST>
|
||||
set(kdt_srcs
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/preprocessor/normalize.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/kdt_parameters.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/index/vector_index/cpu_kdt_rng.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/sptag.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/adapter/arrow.cpp
|
||||
${CORE_SOURCE_DIR}/src/knowhere/common/timer.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/preprocessor/normalize.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/kdt_parameters.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/index/vector_index/cpu_kdt_rng.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/structure.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/sptag.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/exception.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/adapter/arrow.cpp
|
||||
${CORE_SOURCE_DIR}/knowhere/knowhere/common/timer.cpp
|
||||
utils.cpp
|
||||
)
|
||||
if(NOT TARGET test_kdt)
|
||||
|
|
|
|||
|
|
@ -33,14 +33,14 @@ namespace zilliz {
|
|||
namespace milvus {
|
||||
namespace engine {
|
||||
|
||||
Options DBFactory::BuildOption() {
|
||||
DBOptions DBFactory::BuildOption() {
|
||||
auto meta = MetaFactory::BuildOption();
|
||||
Options options;
|
||||
DBOptions options;
|
||||
options.meta = meta;
|
||||
return options;
|
||||
}
|
||||
|
||||
DBPtr DBFactory::Build(const Options& options) {
|
||||
DBPtr DBFactory::Build(const DBOptions& options) {
|
||||
return std::make_shared<DBImpl>(options);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,9 +29,9 @@ namespace engine {
|
|||
|
||||
class DBFactory {
|
||||
public:
|
||||
static Options BuildOption();
|
||||
static DBOptions BuildOption();
|
||||
|
||||
static DBPtr Build(const Options& options);
|
||||
static DBPtr Build(const DBOptions& options);
|
||||
};
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
|
|||
}
|
||||
|
||||
|
||||
DBImpl::DBImpl(const Options& options)
|
||||
DBImpl::DBImpl(const DBOptions& options)
|
||||
: options_(options),
|
||||
shutting_down_(true),
|
||||
compact_thread_pool_(1, 1),
|
||||
|
|
@ -77,7 +77,7 @@ Status DBImpl::Start() {
|
|||
shutting_down_.store(false, std::memory_order_release);
|
||||
|
||||
//for distribute version, some nodes are read only
|
||||
if (options_.mode != Options::MODE::READ_ONLY) {
|
||||
if (options_.mode != DBOptions::MODE::READ_ONLY) {
|
||||
ENGINE_LOG_TRACE << "StartTimerTasks";
|
||||
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
|
||||
}
|
||||
|
|
@ -98,7 +98,7 @@ Status DBImpl::Stop() {
|
|||
//wait compaction/buildindex finish
|
||||
bg_timer_thread_.join();
|
||||
|
||||
if (options_.mode != Options::MODE::READ_ONLY) {
|
||||
if (options_.mode != DBOptions::MODE::READ_ONLY) {
|
||||
meta_ptr_->CleanUp();
|
||||
}
|
||||
|
||||
|
|
@ -684,7 +684,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
|
|||
meta_ptr_->Archive();
|
||||
|
||||
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
|
||||
if (options_.mode == Options::MODE::CLUSTER) {
|
||||
if (options_.mode == DBOptions::MODE::CLUSTER) {
|
||||
ttl = meta::D_SEC;
|
||||
}
|
||||
meta_ptr_->CleanUpFilesWithTTL(ttl);
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class Meta;
|
|||
|
||||
class DBImpl : public DB {
|
||||
public:
|
||||
explicit DBImpl(const Options &options);
|
||||
explicit DBImpl(const DBOptions &options);
|
||||
~DBImpl();
|
||||
|
||||
Status Start() override;
|
||||
|
|
@ -75,28 +75,28 @@ class DBImpl : public DB {
|
|||
Status DropIndex(const std::string& table_id) override;
|
||||
|
||||
Status Query(const std::string &table_id,
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
QueryResults &results) override;
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
QueryResults &results) override;
|
||||
|
||||
Status Query(const std::string &table_id,
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
const meta::DatesT &dates,
|
||||
QueryResults &results) override;
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
const meta::DatesT &dates,
|
||||
QueryResults &results) override;
|
||||
|
||||
Status Query(const std::string &table_id,
|
||||
const std::vector<std::string> &file_ids,
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
const meta::DatesT &dates,
|
||||
QueryResults &results) override;
|
||||
const std::vector<std::string> &file_ids,
|
||||
uint64_t k,
|
||||
uint64_t nq,
|
||||
uint64_t nprobe,
|
||||
const float *vectors,
|
||||
const meta::DatesT &dates,
|
||||
QueryResults &results) override;
|
||||
|
||||
Status Size(uint64_t &result) override;
|
||||
|
||||
|
|
@ -129,7 +129,7 @@ class DBImpl : public DB {
|
|||
Status MemSerialize();
|
||||
|
||||
private:
|
||||
const Options options_;
|
||||
const DBOptions options_;
|
||||
|
||||
std::atomic<bool> shutting_down_;
|
||||
|
||||
|
|
|
|||
|
|
@ -27,9 +27,6 @@ namespace zilliz {
|
|||
namespace milvus {
|
||||
namespace engine {
|
||||
|
||||
Options::Options() {
|
||||
}
|
||||
|
||||
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
|
||||
ParseType(type);
|
||||
ParseCritirias(criterias);
|
||||
|
|
|
|||
|
|
@ -58,16 +58,13 @@ struct DBMetaOptions {
|
|||
ArchiveConf archive_conf = ArchiveConf("delete");
|
||||
}; // DBMetaOptions
|
||||
|
||||
struct Options {
|
||||
|
||||
struct DBOptions {
|
||||
typedef enum {
|
||||
SINGLE,
|
||||
CLUSTER,
|
||||
READ_ONLY
|
||||
} MODE;
|
||||
|
||||
Options();
|
||||
|
||||
uint16_t merge_trigger_number = 2;
|
||||
DBMetaOptions meta;
|
||||
int mode = MODE::SINGLE;
|
||||
|
|
|
|||
|
|
@ -80,17 +80,17 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
|
|||
std::string db_path = options.path;
|
||||
std::string table_path = db_path + TABLES_FOLDER + table_id;
|
||||
auto status = server::CommonUtil::CreateDirectory(table_path);
|
||||
if (status != 0) {
|
||||
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
|
||||
return Status(DB_ERROR, "Failed to create table path");
|
||||
if (!status.ok()) {
|
||||
ENGINE_LOG_ERROR << status.message();
|
||||
return status;
|
||||
}
|
||||
|
||||
for(auto& path : options.slave_paths) {
|
||||
table_path = path + TABLES_FOLDER + table_id;
|
||||
status = server::CommonUtil::CreateDirectory(table_path);
|
||||
if (status != 0) {
|
||||
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
|
||||
return Status(DB_ERROR, "Failed to create table path");
|
||||
if (!status.ok()) {
|
||||
ENGINE_LOG_ERROR << status.message();
|
||||
return status;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -120,9 +120,9 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
|
|||
std::string parent_path = GetTableFileParentFolder(options, table_file);
|
||||
|
||||
auto status = server::CommonUtil::CreateDirectory(parent_path);
|
||||
if (status != 0) {
|
||||
ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error";
|
||||
return Status(DB_ERROR, "Failed to create partition directory");
|
||||
if (!status.ok()) {
|
||||
ENGINE_LOG_ERROR << status.message();
|
||||
return status;
|
||||
}
|
||||
|
||||
table_file.location_ = parent_path + "/" + table_file.file_id_;
|
||||
|
|
@ -148,7 +148,9 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
|
|||
}
|
||||
|
||||
std::string msg = "Table file doesn't exist: " + file_path;
|
||||
ENGINE_LOG_ERROR << msg;
|
||||
ENGINE_LOG_ERROR << msg << " in path: " << options.path
|
||||
<< " for table: " << table_file.table_id_;
|
||||
|
||||
return Status(DB_ERROR, msg);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@
|
|||
#include "utils/CommonUtil.h"
|
||||
#include "utils/Exception.h"
|
||||
|
||||
#include "wrapper/knowhere/vec_index.h"
|
||||
#include "wrapper/knowhere/vec_impl.h"
|
||||
#include "src/wrapper/vec_index.h"
|
||||
#include "src/wrapper/vec_impl.h"
|
||||
#include "knowhere/common/exception.h"
|
||||
|
||||
#include <stdexcept>
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
#pragma once
|
||||
|
||||
#include "ExecutionEngine.h"
|
||||
#include "wrapper/knowhere/vec_index.h"
|
||||
#include "src/wrapper/vec_index.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ class MemManagerImpl : public MemManager {
|
|||
public:
|
||||
using Ptr = std::shared_ptr<MemManagerImpl>;
|
||||
|
||||
MemManagerImpl(const meta::MetaPtr &meta, const Options &options)
|
||||
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
|
||||
: meta_(meta), options_(options) {}
|
||||
|
||||
Status InsertVectors(const std::string &table_id,
|
||||
|
|
@ -66,7 +66,7 @@ class MemManagerImpl : public MemManager {
|
|||
MemIdMap mem_id_map_;
|
||||
MemList immu_mem_list_;
|
||||
meta::MetaPtr meta_;
|
||||
Options options_;
|
||||
DBOptions options_;
|
||||
std::mutex mutex_;
|
||||
std::mutex serialization_mtx_;
|
||||
}; // NewMemManager
|
||||
|
|
|
|||
|
|
@ -31,8 +31,7 @@ namespace zilliz {
|
|||
namespace milvus {
|
||||
namespace engine {
|
||||
|
||||
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta,
|
||||
const Options& options) {
|
||||
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
|
||||
return std::make_shared<MemManagerImpl>(meta, options);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ namespace engine {
|
|||
|
||||
class MemManagerFactory {
|
||||
public:
|
||||
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const Options &options);
|
||||
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ namespace engine {
|
|||
|
||||
MemTable::MemTable(const std::string &table_id,
|
||||
const meta::MetaPtr &meta,
|
||||
const Options &options) :
|
||||
const DBOptions &options) :
|
||||
table_id_(table_id),
|
||||
meta_(meta),
|
||||
options_(options) {
|
||||
|
|
@ -54,7 +54,7 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
|
|||
}
|
||||
|
||||
if (!status.ok()) {
|
||||
std::string err_msg = "MemTable::Add failed: " + status.ToString();
|
||||
std::string err_msg = "Insert failed: " + status.ToString();
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
return Status(DB_ERROR, err_msg);
|
||||
}
|
||||
|
|
@ -74,7 +74,7 @@ Status MemTable::Serialize() {
|
|||
for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) {
|
||||
auto status = (*mem_table_file)->Serialize();
|
||||
if (!status.ok()) {
|
||||
std::string err_msg = "MemTable::Serialize failed: " + status.ToString();
|
||||
std::string err_msg = "Insert data serialize failed: " + status.ToString();
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
return Status(DB_ERROR, err_msg);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ class MemTable {
|
|||
public:
|
||||
using MemTableFileList = std::vector<MemTableFilePtr>;
|
||||
|
||||
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
|
||||
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
|
||||
|
||||
Status Add(VectorSourcePtr &source, IDNumbers &vector_ids);
|
||||
|
||||
|
|
@ -56,7 +56,7 @@ class MemTable {
|
|||
|
||||
meta::MetaPtr meta_;
|
||||
|
||||
Options options_;
|
||||
DBOptions options_;
|
||||
|
||||
std::mutex mutex_;
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ namespace engine {
|
|||
|
||||
MemTableFile::MemTableFile(const std::string &table_id,
|
||||
const meta::MetaPtr &meta,
|
||||
const Options &options) :
|
||||
const DBOptions &options) :
|
||||
table_id_(table_id),
|
||||
meta_(meta),
|
||||
options_(options) {
|
||||
|
|
@ -67,7 +67,7 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
|
|||
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
|
||||
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
|
||||
ENGINE_LOG_ERROR << err_msg;
|
||||
return Status(DB_ERROR, "not able to create table file");
|
||||
return Status(DB_ERROR, "Not able to create table file");
|
||||
}
|
||||
|
||||
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ namespace engine {
|
|||
class MemTableFile {
|
||||
|
||||
public:
|
||||
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
|
||||
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
|
||||
|
||||
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids);
|
||||
|
||||
|
|
@ -53,7 +53,7 @@ class MemTableFile {
|
|||
|
||||
meta::MetaPtr meta_;
|
||||
|
||||
Options options_;
|
||||
DBOptions options_;
|
||||
|
||||
size_t current_mem_;
|
||||
|
||||
|
|
|
|||
|
|
@ -135,7 +135,7 @@ Status MySQLMetaImpl::Initialize() {
|
|||
ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize);
|
||||
try {
|
||||
|
||||
if (mode_ != Options::MODE::READ_ONLY) {
|
||||
if (mode_ != DBOptions::MODE::READ_ONLY) {
|
||||
CleanUp();
|
||||
}
|
||||
|
||||
|
|
@ -621,7 +621,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) {
|
|||
|
||||
} //Scoped Connection
|
||||
|
||||
if (mode_ == Options::MODE::CLUSTER) {
|
||||
if (mode_ == DBOptions::MODE::CLUSTER) {
|
||||
DeleteTableFiles(table_id);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -109,9 +109,9 @@ main(int argc, char *argv[]) {
|
|||
}
|
||||
}
|
||||
|
||||
server::Server* server_ptr = server::Server::Instance();
|
||||
server_ptr->Init(start_daemonized, pid_filename, config_filename, log_config_file);
|
||||
return server_ptr->Start();
|
||||
server::Server& server = server::Server::Instance();
|
||||
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
|
||||
return server.Start();
|
||||
}
|
||||
|
||||
void
|
||||
|
|
|
|||
|
|
@ -20,23 +20,10 @@
|
|||
|
||||
aux_source_directory(interface interface_files)
|
||||
|
||||
include_directories(/usr/include)
|
||||
include_directories(include)
|
||||
include_directories(/usr/local/include)
|
||||
|
||||
|
||||
aux_source_directory(grpc grpc_client_files)
|
||||
|
||||
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus)
|
||||
include_directories(${CMAKE_SOURCE_DIR}/src/grpc/gen-status)
|
||||
|
||||
set(grpc_service_files
|
||||
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.grpc.pb.cc
|
||||
${CMAKE_SOURCE_DIR}/src/grpc/gen-milvus/milvus.pb.cc
|
||||
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.grpc.pb.cc
|
||||
${CMAKE_SOURCE_DIR}/src/grpc/gen-status/status.pb.cc
|
||||
)
|
||||
|
||||
add_library(milvus_sdk STATIC
|
||||
${interface_files}
|
||||
${grpc_client_files}
|
||||
|
|
|
|||
|
|
@ -20,11 +20,6 @@
|
|||
|
||||
aux_source_directory(src src_files)
|
||||
|
||||
include_directories(src)
|
||||
include_directories(../../include)
|
||||
|
||||
link_directories(${CMAKE_BINARY_DIR})
|
||||
|
||||
add_executable(sdk_simple
|
||||
main.cpp
|
||||
${src_files}
|
||||
|
|
|
|||
|
|
@ -190,7 +190,7 @@ void DoSearch(std::shared_ptr<Connection> conn,
|
|||
{
|
||||
TimeRecorder rc(phase_name);
|
||||
Status stat = conn->Search(TABLE_NAME, record_array, query_range_array, TOP_K, 32, topk_query_result_array);
|
||||
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
auto finish = std::chrono::high_resolution_clock::now();
|
||||
std::cout << "SEARCHVECTOR COST: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
|
||||
|
|
@ -207,7 +207,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
{//connect server
|
||||
ConnectParam param = {address, port};
|
||||
Status stat = conn->Connect(param);
|
||||
std::cout << "Connect function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "Connect function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
|
||||
{//server version
|
||||
|
|
@ -223,7 +223,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
{
|
||||
std::vector<std::string> tables;
|
||||
Status stat = conn->ShowTables(tables);
|
||||
std::cout << "ShowTables function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "ShowTables function call status: " << stat.message() << std::endl;
|
||||
std::cout << "All tables: " << std::endl;
|
||||
for(auto& table : tables) {
|
||||
int64_t row_count = 0;
|
||||
|
|
@ -236,7 +236,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
{//create table
|
||||
TableSchema tb_schema = BuildTableSchema();
|
||||
Status stat = conn->CreateTable(tb_schema);
|
||||
std::cout << "CreateTable function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "CreateTable function call status: " << stat.message() << std::endl;
|
||||
PrintTableSchema(tb_schema);
|
||||
|
||||
bool has_table = conn->HasTable(tb_schema.table_name);
|
||||
|
|
@ -248,7 +248,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
{//describe table
|
||||
TableSchema tb_schema;
|
||||
Status stat = conn->DescribeTable(TABLE_NAME, tb_schema);
|
||||
std::cout << "DescribeTable function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "DescribeTable function call status: " << stat.message() << std::endl;
|
||||
PrintTableSchema(tb_schema);
|
||||
}
|
||||
|
||||
|
|
@ -279,7 +279,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
std::cout << "InsertVector cost: " << std::chrono::duration_cast<std::chrono::duration<double>>(finish - start).count() << "s\n";
|
||||
|
||||
|
||||
std::cout << "InsertVector function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "InsertVector function call status: " << stat.message() << std::endl;
|
||||
std::cout << "Returned id array count: " << record_ids.size() << std::endl;
|
||||
|
||||
if(search_record_array.size() < NQ) {
|
||||
|
|
@ -305,16 +305,16 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
index.index_type = IndexType::gpu_ivfsq8;
|
||||
index.nlist = 16384;
|
||||
Status stat = conn->CreateIndex(index);
|
||||
std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
|
||||
|
||||
IndexParam index2;
|
||||
stat = conn->DescribeIndex(TABLE_NAME, index2);
|
||||
std::cout << "DescribeIndex function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
|
||||
{//preload table
|
||||
Status stat = conn->PreloadTable(TABLE_NAME);
|
||||
std::cout << "PreloadTable function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "PreloadTable function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
|
||||
{//search vectors after build index finish
|
||||
|
|
@ -326,7 +326,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
|
||||
{//delete index
|
||||
Status stat = conn->DropIndex(TABLE_NAME);
|
||||
std::cout << "DropIndex function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
|
||||
|
||||
int64_t row_count = 0;
|
||||
stat = conn->CountTable(TABLE_NAME, row_count);
|
||||
|
|
@ -339,12 +339,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
|
|||
rg.end_value = CurrentTmDate(-3);
|
||||
|
||||
Status stat = conn->DeleteByRange(rg, TABLE_NAME);
|
||||
std::cout << "DeleteByRange function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "DeleteByRange function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
|
||||
{//delete table
|
||||
Status stat = conn->DropTable(TABLE_NAME);
|
||||
std::cout << "DeleteTable function call status: " << stat.ToString() << std::endl;
|
||||
std::cout << "DeleteTable function call status: " << stat.message() << std::endl;
|
||||
}
|
||||
|
||||
{//server status
|
||||
|
|
|
|||
|
|
@ -40,14 +40,12 @@ enum class StatusCode {
|
|||
ServerFailed,
|
||||
};
|
||||
|
||||
using ErrorCode = StatusCode;
|
||||
|
||||
/**
|
||||
* @brief Status for SDK interface return
|
||||
*/
|
||||
class Status {
|
||||
public:
|
||||
Status(ErrorCode code, const std::string &msg);
|
||||
Status(StatusCode code, const std::string &msg);
|
||||
Status();
|
||||
~Status();
|
||||
|
||||
|
|
@ -67,14 +65,14 @@ public:
|
|||
bool
|
||||
ok() const { return state_ == nullptr || code() == StatusCode::OK; }
|
||||
|
||||
std::string
|
||||
ToString() const;
|
||||
|
||||
ErrorCode
|
||||
StatusCode
|
||||
code() const {
|
||||
return (state_ == nullptr) ? StatusCode::OK : *(ErrorCode*)(state_);
|
||||
return (state_ == nullptr) ? StatusCode::OK : *(StatusCode*)(state_);
|
||||
}
|
||||
|
||||
std::string
|
||||
message() const;
|
||||
|
||||
private:
|
||||
inline void
|
||||
CopyFrom(const Status &s);
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue