milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Integreation with Velox (#22102) 

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
pull/22232/head
Enwei Jiao 2023-02-16 17:26:35 +08:00 committed by GitHub
parent 187788059b
commit b25b3ef431
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 143 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.env
View File

@ -1,8 +1,8 @@
IMAGE_REPO=milvusdb
IMAGE_ARCH=amd64
OS_NAME=ubuntu20.04
DATE_VERSION=20230209-e1ec248
LATEST_DATE_VERSION=20230209-e1ec248
DATE_VERSION=20230215-5351fee
LATEST_DATE_VERSION=20230215-5351fee
MINIO_ADDRESS=minio:9000
PULSAR_ADDRESS=pulsar://pulsar:6650
ETCD_ENDPOINTS=etcd:2379

20
.github/workflows/code-checker.yaml vendored
View File

@ -27,7 +27,7 @@ jobs:
ubuntu:
name: ${{ matrix.name }}
runs-on: ${{ matrix.os }}
timeout-minutes: 60
timeout-minutes: 120
strategy:
fail-fast: false
matrix:
@ -46,7 +46,8 @@ jobs:
echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
echo "Set CCache hash to ${CORE_HASH}"
- name: Cache CCache Volumes
uses: actions/cache@v3
# uses: actions/cache@v3
uses: pat-s/always-upload-cache@v3
with:
path: .docker/amd64-ubuntu20.04-ccache
key: ubuntu20.04-ccache-${{ env.corehash }}
@ -58,7 +59,8 @@ jobs:
key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }}
restore-keys: ubuntu20.04-go-mod-
- name: Cache Conan Packages
uses: actions/cache@v3
# uses: actions/cache@v3
uses: pat-s/always-upload-cache@v3
with:
path: .docker/amd64-ubuntu20.04-conan
key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }}
@ -72,7 +74,7 @@ jobs:
name: Code Checker CentOS 7
# Run in centos docker
runs-on: ubuntu-20.04
timeout-minutes: 60
timeout-minutes: 120
steps:
- name: Checkout
uses: actions/checkout@v2
@ -83,7 +85,8 @@ jobs:
echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
echo "Set CCache hash to ${CORE_HASH}"
- name: Cache CCache Volumes
uses: actions/cache@v3
# uses: actions/cache@v3
uses: pat-s/always-upload-cache@v3
with:
path: .docker/amd64-centos7-ccache
key: centos7-ccache-${{ env.corehash }}
@ -101,11 +104,12 @@ jobs:
key: centos7-go-mod-${{ hashFiles('**/go.sum') }}
restore-keys: centos7-go-mod-
- name: Cache Conan Packages
uses: actions/cache@v3
# uses: actions/cache@v3
uses: pat-s/always-upload-cache@v3
with:
path: .docker/amd64-centos7-conan
key: centos7-conan-${{ hashFiles('internal/core/conanfile.txt') }}
restore-keys: centos7-conan-
key: centos7-conan-${{ hashFiles('internal/core/conanfile.*') }}
# restore-keys: centos7-conan-
- name: Code Check
env:
OS_NAME: 'centos7'

5
.github/workflows/mac.yaml vendored
View File

@ -26,7 +26,7 @@ jobs:
mac:
name: Code Checker MacOS 12
runs-on: macos-12
timeout-minutes: 120
timeout-minutes: 180
steps:
- name: Checkout
uses: actions/checkout@v2
@ -37,7 +37,8 @@ jobs:
echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
echo "Set CCache hash to ${CORE_HASH}"
- name: Mac Cache Third Party
uses: actions/cache@v3
# uses: actions/cache@v3
uses: pat-s/always-upload-cache@v3
with:
path: |
/tmp/thirdparty

9
.github/workflows/main.yaml vendored
View File

@ -1,5 +1,4 @@
name: Build and test
# TODO: do not trigger action for some document file update
# This workflow is triggered on pushes or pull request to the repository.
on:
@ -36,7 +35,7 @@ jobs:
UT:
name: Build and test AMD64 Ubuntu ${{ matrix.ubuntu }}
runs-on: ubuntu-${{ matrix.ubuntu }}
timeout-minutes: 120
timeout-minutes: 180
strategy:
fail-fast: false
matrix:
@ -60,12 +59,12 @@ jobs:
**/*.h
**/*.hpp
**/*.CMakeLists.txt
**/conanfile.*
**/conanfile.*
- name: 'Setup Use USE_ASAN'
if: steps.changed-files-cpp.outputs.any_changed == 'true'
run: |
echo "useasan=true" >> $GITHUB_ENV
echo "Setup USE_ASAN to true since cpp file(s) changed"
echo "useasan=true" >> $GITHUB_ENV
echo "Setup USE_ASAN to true since cpp file(s) changed"
- name: 'Generate CCache Hash'
env:
CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt', 'internal/core/conanfile.py') }}

4
internal/core/CMakeLists.txt
View File

@ -19,8 +19,6 @@ cmake_minimum_required( VERSION 3.18 )
if ( APPLE )
set( CMAKE_CROSSCOMPILING TRUE )
set( RUN_HAVE_GNU_POSIX_REGEX 0 )
set( CMAKE_C_COMPILER "/usr/local/opt/llvm/bin/clang" )
set( CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++" )
endif ()
add_definitions(-DELPP_THREAD_SAFE)
@ -31,7 +29,6 @@ project(core)
include(CheckCXXCompilerFlag)
if ( APPLE )
message(STATUS "==============Darwin Environment==============")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/opt/llvm/include -I/usr/local/include -I/usr/local/opt/libomp/include -L/usr/local/opt/libomp/lib")
elseif (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
message(STATUS "==============Linux Environment===============")
set(LINUX TRUE)
@ -90,7 +87,6 @@ set( CONAN_DISABLE_CHECK_COMPILER ON )
conan_basic_setup( KEEP_RPATHS )
set( CMAKE_PREFIX_PATH ${CMAKE_PREFIX_PATH} ${CONAN_PROTOBUF_ROOT})
message( STATUS "cmake prefix path ${CMAKE_PREFIX_PATH}")
include( CTest )
include( BuildUtils )

23
internal/core/conanfile.py
View File

@ -9,6 +9,9 @@ class MilvusConan(ConanFile):
"boost/1.81.0",
"onetbb/2021.7.0",
"zstd/1.5.2",
"lz4/1.9.4",
"snappy/1.1.9",
"lzo/2.10",
"arrow/8.0.1",
"openssl/1.1.1q",
"aws-sdk-cpp/1.9.234",
@ -20,8 +23,20 @@ class MilvusConan(ConanFile):
"marisa/0.2.6",
"zlib/1.2.13",
"libcurl/7.86.0",
"glog/0.6.0",
"fmt/8.0.1",
"gflags/2.2.2",
"double-conversion/3.2.1",
"libevent/2.1.12",
"libdwarf/20191104",
"libiberty/9.1.0",
"libsodium/cci.20220430",
"bison/3.5.3",
"flex/2.6.4",
"xsimd/9.0.1",
"folly/2022.10.31.01@milvus/dev",
"velox/2023.02.07.01@milvus/dev",
)
generators = ("cmake", "cmake_find_package")
default_options = {
"rocksdb:shared": True,
@ -33,14 +48,16 @@ class MilvusConan(ConanFile):
"aws-sdk-cpp:text-to-speech": False,
"aws-sdk-cpp:transfer": False,
"gtest:build_gmock": False,
"folly:use_sse4_2": True,
}
should_build = False
def configure(self):
# Macos M1 cannot use jemalloc
if self.settings.os == "Macos":
if self.settings.arch not in ("x86_64", "x86"):
self.options["arrow"].with_jemalloc = False
del self.options["folly"].use_sse4_2
self.options["arrow"].with_jemalloc = False
self.options["boost"].without_fiber = True
self.options["boost"].without_json = True
self.options["boost"].without_wave = True

2
internal/core/src/common/CMakeLists.txt
View File

@ -28,8 +28,6 @@ target_link_libraries(milvus_common
milvus_log
yaml-cpp
boost_bitset_ext
# arrow
# parquet
${CONAN_LIBS}
)

4
internal/core/src/segcore/CMakeLists.txt
View File

@ -53,3 +53,7 @@ target_link_libraries(milvus_segcore
)
install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}")
add_executable(velox_demo VeloxDemo.cpp)
target_link_libraries(velox_demo ${CONAN_LIBS} velox_bundled)
install(TARGETS velox_demo DESTINATION "${CMAKE_INSTALL_BINDIR}")

66
internal/core/src/segcore/VeloxDemo.cpp Normal file
View File

@ -0,0 +1,66 @@
/*
* Copyright (c) Facebook, Inc. and its affiliates.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <folly/init/Init.h>
#include <algorithm>
#include "velox/common/file/FileSystems.h"
#include "velox/common/memory/Memory.h"
#include "velox/dwio/dwrf/reader/DwrfReader.h"
#include "velox/exec/tests/utils/TempDirectoryPath.h"
#include "velox/vector/BaseVector.h"
using namespace facebook::velox;
using namespace facebook::velox::dwio::common;
using namespace facebook::velox::dwrf;
// A temporary program that reads from ORC file and prints its content
// Used to compare the ORC data read by DWRFReader against apache-orc repo.
// Usage: velox_example_scan_orc {orc_file_path}
int
main(int argc, char** argv) {
folly::init(&argc, &argv);
if (argc < 2) {
return 1;
}
// To be able to read local files, we need to register the local file
// filesystem. We also need to register the dwrf reader factory:
filesystems::registerLocalFileSystem();
dwrf::registerDwrfReaderFactory();
auto pool = facebook::velox::memory::getDefaultMemoryPool();
std::string filePath{argv[1]};
ReaderOptions readerOpts{pool.get()};
// To make DwrfReader reads ORC file, setFileFormat to FileFormat::ORC
readerOpts.setFileFormat(FileFormat::ORC);
auto reader = DwrfReader::create(
std::make_unique<BufferedInput>(std::make_shared<LocalReadFile>(filePath), readerOpts.getMemoryPool()),
readerOpts);
VectorPtr batch;
RowReaderOptions rowReaderOptions;
auto rowReader = reader->createRowReader(rowReaderOptions);
while (rowReader->next(500, batch)) {
auto rowVector = batch->as<RowVector>();
for (vector_size_t i = 0; i < rowVector->size(); ++i) {
std::cout << rowVector->toString(i) << std::endl;
}
}
return 0;
}

40
internal/core/thirdparty/CMakeLists.txt vendored
View File

@ -12,37 +12,37 @@
#-------------------------------------------------------------------------------
# Using default c and cxx compiler in our build tree
# Thirdpart cxx and c flags
add_compile_options( -O3 -fPIC -Wno-error -fopenmp )
add_compile_options(-O3 -fPIC -Wno-error -fopenmp)
if ( NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD )
set( EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1 )
if (NOT KNOWHERE_VERBOSE_THIRDPARTY_BUILD)
set(EP_LOG_OPTIONS LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1 LOG_DOWNLOAD 1)
else ()
set( EP_LOG_OPTIONS )
set(EP_LOG_OPTIONS)
endif ()
set( MAKE_BUILD_ARGS "-j6" )
set(MAKE_BUILD_ARGS "-j6")
include( FetchContent )
set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
set( FETCHCONTENT_QUIET OFF )
include(FetchContent)
set(FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download)
set(FETCHCONTENT_QUIET OFF)
if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
if(CUSTOM_THIRDPARTY_DOWNLOAD_PATH)
set(THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH})
else()
set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
set(THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download)
endif()
message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
message(STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}")
# ----------------------------------------------------------------------
# Find pthreads
set( THREADS_PREFER_PTHREAD_FLAG ON )
find_package( Threads REQUIRED )
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
add_subdirectory( knowhere )
add_subdirectory(knowhere)
add_subdirectory( boost_ext )
add_subdirectory( rocksdb )
add_subdirectory(boost_ext)
add_subdirectory(rocksdb)
if ( LINUX )
add_subdirectory( jemalloc )
endif()
if (LINUX)
add_subdirectory(jemalloc)
endif()

17
scripts/core_build.sh
View File

@ -188,6 +188,8 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then
exit 0
fi
export CONAN_REVISIONS_ENABLED=1
conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local
unameOut="$(uname -s)"
case "${unameOut}" in
Darwin*)
@ -195,15 +197,18 @@ case "${unameOut}" in
export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
export CC="${llvm_prefix}/bin/clang"
export CXX="${llvm_prefix}/bin/clang++"
export LDFLAGS="-L${llvm_prefix}/lib -L/usr/local/opt/libomp/lib"
export CXXFLAGS="-I${llvm_prefix}/include -I/usr/local/include -I/usr/local/opt/libomp/include"
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; }
export CFLAGS=-Wno-deprecated-declarations
export CXXFLAGS=-Wno-deprecated-declarations
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=15 -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; }
;;
Linux*)
echo "Running on ${OS_NAME}"
export CPU_TARGET=avx
GCC_VERSION=`gcc -dumpversion`
if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
else
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
fi
;;
*)
@ -211,6 +216,7 @@ case "${unameOut}" in
;;
esac
arch=$(uname -m)
CMAKE_CMD="cmake \
${CMAKE_EXTRA_ARGS} \
-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
@ -218,6 +224,7 @@ ${CMAKE_EXTRA_ARGS} \
-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
-DOpenBLAS_SOURCE=AUTO \
-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
-DCMAKE_LIBRARY_ARCHITECTURE=${arch} \
-DBUILD_COVERAGE=${BUILD_COVERAGE} \
-DMILVUS_DB_PATH=${DB_PATH} \
-DENABLE_CPU_PROFILING=${PROFILING} \