milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

move arrow from storage to core (#17061) 

Signed-off-by: Enwei Jiao <jiaoew2011@gmail.com>
pull/17149/head
Enwei Jiao 2022-05-22 20:03:58 +08:00 committed by GitHub
parent e8df5ed406
commit d28a2db46c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 154 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
go.mod
View File

@ -6,8 +6,8 @@ require (
github.com/BurntSushi/toml v1.0.0
github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
github.com/antonmedv/expr v1.8.9
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7
github.com/apache/pulsar-client-go/oauth2 v0.0.0-20201120111947-b8bd55bc02bd // indirect
@ -65,4 +65,4 @@ replace (
github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt v3.2.2+incompatible // Fix security alert for jwt-go 3.2.0
github.com/go-kit/kit => github.com/go-kit/kit v0.1.0
google.golang.org/grpc => google.golang.org/grpc v1.38.0
)
)

4
internal/core/CMakeLists.txt
View File

@ -84,9 +84,9 @@ using_ccache_if_defined(MILVUS_USE_CCACHE)
include( ExternalProject )
include( GNUInstallDirs )
include( FetchContent )
include_directories(thirdparty)
include_directories( thirdparty )
set( FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download )
set(FETCHCONTENT_QUIET OFF)
set( FETCHCONTENT_QUIET OFF )
include( ThirdPartyPackages )
# **************************** Compiler arguments ****************************

27
internal/core/src/common/CMakeLists.txt
View File

@ -20,6 +20,31 @@ add_library(milvus_common SHARED
${COMMON_SRC}
)
target_link_libraries(milvus_common milvus_utils milvus_config milvus_log knowhere milvus_proto yaml-cpp boost_bitset_ext)
if ( MSYS )
target_link_libraries(milvus_common
milvus_utils
milvus_config
milvus_log
knowhere
milvus_proto
yaml-cpp
boost_bitset_ext
arrow
parquet
)
else()
target_link_libraries(milvus_common
milvus_utils
milvus_config
milvus_log
knowhere
milvus_proto
yaml-cpp
boost_bitset_ext
arrow
parquet
arrow_bundled
)
endif()
install(TARGETS milvus_common DESTINATION lib)

1
internal/core/thirdparty/CMakeLists.txt vendored
View File

@ -61,6 +61,7 @@ endif()
add_subdirectory( protobuf )
add_subdirectory( boost_ext )
add_subdirectory( arrow )
# ******************************* Thridparty marisa ********************************
# TODO: support apple & win.

111
internal/core/thirdparty/arrow/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,111 @@
# Licensed to the LF AI & Data foundation under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set( ARROW_VERSION "8.0.0" )
set( ARROW_SOURCE_URL
"https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz")
set( ARROW_MD5 "b1d285d2b90feceb61155baf9a21f81c" )
# These three components is required by arrow
find_package(Boost REQUIRED COMPONENTS regex system filesystem)
message( STATUS "Find Boost: include dirs-${Boost_INCLUDE_DIRS}, version-${Boost_VERSION}")
macro( build_arrow )
message( STATUS "Building ARROW-${ARROW_VERSION} from source" )
set( ARROW_CMAKE_ARGS
"-DARROW_WITH_LZ4=OFF"
"-DARROW_WITH_ZSTD=ON"
"-Dzstd_SOURCE=BUNDLED"
"-DARROW_WITH_BROTLI=OFF"
"-DARROW_WITH_SNAPPY=OFF"
"-DARROW_WITH_ZLIB=OFF"
"-DARROW_BUILD_STATIC=ON"
"-DARROW_BUILD_SHARED=OFF"
"-DARROW_BOOST_USE_SHARED=OFF"
"-DARROW_BUILD_TESTS=OFF"
"-DARROW_TEST_MEMCHECK=OFF"
"-DARROW_BUILD_BENCHMARKS=OFF"
"-DARROW_CUDA=OFF"
"-DARROW_PYTHON=OFF"
"-DARROW_WITH_RE2=OFF"
"-DARROW_BUILD_UTILITIES=OFF"
"-DARROW_PARQUET=ON"
"-DPARQUET_BUILD_SHARED=OFF"
"-DThrift_SOURCE=BUNDLED"
"-Dutf8proc_SOURCE=BUNDLED"
"-DARROW_S3=OFF"
"-DCMAKE_VERBOSE_MAKEFILE=ON"
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
"-DCMAKE_INCLUDE_PATH=${Boost_INCLUDE_DIRS}"
)
if (WIN32)
set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=OFF" )
else ()
set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=ON" )
endif ()
ExternalProject_Add(
arrow-ep
PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/arrow-subbuild
BINARY_DIR arrow-bin
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
INSTALL_DIR ${CMAKE_INSTALL_PREFIX}
SOURCE_SUBDIR "cpp"
URL ${ARROW_SOURCE_URL}
URL_MD5 ${ARROW_MD5}
CMAKE_ARGS ${ARROW_CMAKE_ARGS}
${EP_LOG_OPTIONS}
)
ExternalProject_Get_Property( arrow-ep INSTALL_DIR )
if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
endif()
add_library( arrow_bundled STATIC IMPORTED )
set_target_properties( arrow_bundled
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow_bundled_dependencies.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(arrow_bundled arrow-ep )
add_library( arrow STATIC IMPORTED )
set_target_properties( arrow
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(arrow arrow_bundled)
target_link_libraries(arrow INTERFACE arrow_bundled)
add_library( parquet STATIC IMPORTED )
set_target_properties( parquet
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libparquet.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies( parquet arrow-ep )
target_link_libraries( parquet INTERFACE arrow )
endmacro()
if (MSYS)
message(STATUS "Using arrow form system")
else (MSYS)
build_arrow()
endif (MSYS)

110
internal/storage/cwrapper/CMakeLists.txt
View File

@ -22,111 +22,19 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
include( GNUInstallDirs )
include( ExternalProject )
set( ARROW_VERSION "6.0.1" )
set( ARROW_SOURCE_URL
"https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz")
if( CUSTOM_THIRDPARTY_DOWNLOAD_PATH )
set( THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} )
else()
set( THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download )
endif()
message( STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}" )
# These three components is required by arrow
find_package(Boost REQUIRED COMPONENTS regex system filesystem)
message( STATUS "Find Boost: include dirs-${Boost_INCLUDE_DIRS}, version-${Boost_VERSION}")
macro( build_arrow )
message( STATUS "Building ARROW-${ARROW_VERSION} from source" )
set( ARROW_CMAKE_ARGS
"-DARROW_WITH_LZ4=OFF"
"-DARROW_WITH_ZSTD=ON"
"-Dzstd_SOURCE=BUNDLED"
"-DARROW_WITH_BROTLI=OFF"
"-DARROW_WITH_SNAPPY=OFF"
"-DARROW_WITH_ZLIB=OFF"
"-DARROW_BUILD_STATIC=ON"
"-DARROW_BUILD_SHARED=OFF"
"-DARROW_BOOST_USE_SHARED=OFF"
"-DARROW_BUILD_TESTS=OFF"
"-DARROW_TEST_MEMCHECK=OFF"
"-DARROW_BUILD_BENCHMARKS=OFF"
"-DARROW_CUDA=OFF"
"-DARROW_PYTHON=OFF"
"-DARROW_WITH_RE2=OFF"
"-DARROW_BUILD_UTILITIES=OFF"
"-DARROW_PARQUET=ON"
"-DPARQUET_BUILD_SHARED=OFF"
"-DThrift_SOURCE=BUNDLED"
"-Dutf8proc_SOURCE=BUNDLED"
"-DARROW_S3=OFF"
"-DCMAKE_VERBOSE_MAKEFILE=ON"
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
"-DCMAKE_INCLUDE_PATH=${Boost_INCLUDE_DIRS}"
)
if (WIN32)
set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=OFF" )
else ()
set( ARROW_CMAKE_ARGS ${ARROW_CMAKE_ARGS} "-DARROW_JEMALLOC=ON" )
endif ()
ExternalProject_Add(
arrow-ep
PREFIX ${CMAKE_BINARY_DIR}/3rdparty_download/arrow-subbuild
BINARY_DIR arrow-bin
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}
INSTALL_DIR ${CMAKE_INSTALL_PREFIX}
SOURCE_SUBDIR "cpp"
URL ${ARROW_SOURCE_URL}
URL_MD5 "e00927820d0350879c2abbc77e72c863"
CMAKE_ARGS ${ARROW_CMAKE_ARGS}
${EP_LOG_OPTIONS}
)
ExternalProject_Get_Property( arrow-ep INSTALL_DIR )
if( NOT IS_DIRECTORY ${INSTALL_DIR}/include )
file( MAKE_DIRECTORY "${INSTALL_DIR}/include" )
endif()
add_library( arrow_bundled STATIC IMPORTED )
set_target_properties( arrow_bundled
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow_bundled_dependencies.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(arrow_bundled arrow-ep )
add_library( arrow STATIC IMPORTED )
set_target_properties( arrow
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libarrow.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(arrow arrow_bundled)
target_link_libraries(arrow INTERFACE arrow_bundled)
add_library( parquet STATIC IMPORTED )
set_target_properties( parquet
PROPERTIES
IMPORTED_GLOBAL TRUE
IMPORTED_LOCATION ${INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libparquet.a
INTERFACE_INCLUDE_DIRECTORIES ${INSTALL_DIR}/include )
add_dependencies(parquet arrow-ep)
target_link_libraries(parquet INTERFACE arrow )
endmacro()
if (MSYS)
message(STATUS "Using arrow form system")
else (MSYS)
build_arrow()
endif (MSYS)
include( CheckCXXCompilerFlag )
set(ARROW_OUTPUT_PREFIX ${CMAKE_INSTALL_PREFIX}/../../../core/output)
add_library(wrapper STATIC)
target_sources(wrapper PUBLIC ParquetWrapper.cpp PayloadStream.cpp)
if ( NOT MSYS )
add_library(parquet STATIC IMPORTED)
set_target_properties(parquet PROPERTIES
IMPORTED_LOCATION "${ARROW_OUTPUT_PREFIX}/lib/libparquet.a"
INTERFACE_INCLUDE_DIRECTORIES "${ARROW_OUTPUT_PREFIX}/include")
endif()
if ( EMBEDDED_MILVUS )
message ( STATUS "Turning on fPIC while building embedded Milvus" )
set_target_properties( wrapper PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} POSITION_INDEPENDENT_CODE ON )

6
internal/storage/payload_reader_cgo.go
View File

@ -3,9 +3,9 @@ package storage
/*
#cgo CFLAGS: -I${SRCDIR}/cwrapper
#cgo linux LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -L${SRCDIR}/cwrapper/output/lib64 -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm
#cgo darwin LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm
#cgo windows LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -lwrapper -lparquet -larrow -lstdc++
#cgo linux LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -L${SRCDIR}/cwrapper/output/lib64 -L${SRCDIR}/../core/output/lib -L${SRCDIR}/../core/output/lib64 -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm
#cgo darwin LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -L${SRCDIR}/../core/output/lib -lwrapper -lparquet -larrow -larrow_bundled_dependencies -lstdc++ -lm
#cgo windows LDFLAGS: -L${SRCDIR}/cwrapper/output/lib -L${SRCDIR}/../core/output/lib -lwrapper -lparquet -larrow -lstdc++
#include <stdlib.h>
#include "ParquetWrapper.h"
*/