milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable UnitTest 

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
pull/4973/head^2
quicksilver 2020-10-27 15:51:16 +08:00 committed by yefu.chen
parent 77b2fcf015
commit d66d48c6b6
19 changed files with 242 additions and 374 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.env Normal file
View File

@ -0,0 +1,3 @@
REPO=milvusdb/milvus-distributed-dev
ARCH=amd64
UBUNTU=18.04

72
.github/workflows/main.yaml vendored
View File

@ -9,7 +9,7 @@ on:
- 'scripts/**'
- 'internal/**'
- 'cmd/**'
- '.github/workflows/main.yml'
- '.github/workflows/main.yaml'
- docker-compose.yml
- '!**.md'
- '!**_test.go'
@ -19,48 +19,58 @@ on:
- 'scripts/**'
- 'internal/**'
- 'cmd/**'
- '.github/workflows/main.yml'
- '.github/workflows/main.yaml'
- docker-compose.yml
- '!**.md'
- '!**_test.go'
jobs:
ubuntu:
name: AMD64 ubuntu-18.04
runs-on: ubuntu-18.04
name: AMD64 Ubuntu ${{ matrix.ubuntu }}
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
ubuntu: [18.04]
env:
UBUNTU: ${{ matrix.ubuntu }}
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install Dependency
run: |
./scripts/install_deps.sh
go get github.com/golang/protobuf/protoc-gen-go@v1.3.2
- name: Cache Core Thirdparty
id: cache-core
uses: actions/cache@v2
- name: Check Dockerfile
uses: reviewdog/action-hadolint@v1
with:
path: |
./internal/core/cmake_build
key: ${{ runner.os }}-core-thirdparty
- name: Build Cpp
github_token: ${{ secrets.GITHUB_TOKEN }}
reporter: github-pr-check # Default is github-pr-check
- name: Docker Pull
shell: bash
run: |
./scripts/core_build.sh -u
- name: Generat Proto GO File
docker-compose pull --ignore-pull-failures ubuntu
- name: Docker Build
shell: bash
run: |
echo `pwd`
pwd_dir=`pwd`
export PATH=$PATH:$(go env GOPATH)/bin
export protoc=${pwd_dir}/internal/core/cmake_build/thirdparty/protobuf/protobuf-build/protoc
./scripts/proto_gen_go.sh
- name: Build GO
docker-compose build ubuntu
docker rmi $(docker images | grep '<none>' | awk '{print $3}') || exit 0
- name: Cache Docker Volumes
uses: actions/cache@v1
with:
path: .docker
key: ubuntu${{ matrix.ubuntu }}-${{ hashFiles('internal/core/**') }}
restore-keys: ubuntu${{ matrix.ubuntu }}-
- name: Docker Run
run: |
go build -o ./cmd/writer/writer ./cmd/writer/writer.go
go build -o ./cmd/reader/reader ./cmd/reader/reader.go
go build -o ./cmd/master/master ./cmd/master/master.go
go build -o ./cmd/proxy/proxy ./cmd/proxy/proxy.go
- name: Docker Pull And Run
docker-compose run ubuntu
- name: Run UnitTest
shell: bash
run: |
docker-compose up -d
- name: Run Unittest
cd ${GITHUB_WORKSPACE}/deployments/docker && docker-compose up -d
cd ${GITHUB_WORKSPACE}/scripts/ && ./run_unittest.sh
- name: Docker Push
if: success() && github.event_name == 'push' && github.repository == 'zilliztech/milvus-distributed'
continue-on-error: true
shell: bash
run: |
./scripts/run_unittest.sh
docker login -u ${{ secrets.DOCKERHUB_USER }} \
-p ${{ secrets.DOCKERHUB_TOKEN }}
docker-compose push ubuntu

2
build/docker/build_env/cpu/ubuntu18.04/Dockerfile → build/docker/env/cpu/ubuntu18.04/Dockerfile vendored
View File

@ -42,7 +42,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.15.2.linux-am
go get github.com/golang/protobuf/protoc-gen-go@v1.3.2
# Set permissions on /etc/passwd and /home to allow arbitrary users to write
COPY --chown=0:0 docker/build_env/entrypoint.sh /
COPY --chown=0:0 build/docker/env/entrypoint.sh /
RUN mkdir -p /home/user && chgrp -R 0 /home && chmod -R g=u /etc/passwd /etc/group /home && chmod +x /entrypoint.sh
ENV HOME=/home/user

0
build/docker/build_env/entrypoint.sh → build/docker/env/entrypoint.sh vendored
View File

59
deployments/docker/docker-compose.yml Normal file
View File

@ -0,0 +1,59 @@
version: '3.5'
services:
etcd:
image: quay.io/coreos/etcd:latest
command: etcd -listen-peer-urls=http://127.0.0.1:12380 -advertise-client-urls=http://127.0.0.1:12379 -listen-client-urls http://0.0.0.0:12379,http://0.0.0.0:14001 -initial-advertise-peer-urls=http://127.0.0.1:12380 --initial-cluster default=http://127.0.0.1:12380
ports:
- "12379:12379"
- "12380:12380"
- "14001:14001"
pulsar:
image: apachepulsar/pulsar:latest
command: bin/pulsar standalone
ports:
- "6650:6650"
- "18080:8080"
pd0:
image: pingcap/pd:latest
network_mode: "host"
ports:
- "2379:2379"
- "2380:2380"
volumes:
- /tmp/config/pd.toml:/pd.toml:ro
- /tmp/data:/data
- /tmp/logs:/logs
- /etc/localtime:/etc/localtime:ro
command:
- --name=pd0
- --client-urls=http://0.0.0.0:2379
- --peer-urls=http://0.0.0.0:2380
- --advertise-client-urls=http://127.0.0.1:2379
- --advertise-peer-urls=http://127.0.0.1:2380
- --initial-cluster=pd0=http://127.0.0.1:2380
- --data-dir=/data/pd0
- --log-file=/logs/pd0.log
restart: on-failure
tikv0:
network_mode: "host"
image: pingcap/tikv:latest
ports:
- "20160:20160"
volumes:
- /tmp/config/tikv.toml:/tikv.toml:ro
- /tmp/data:/data
- /tmp/logs:/logs
- /etc/localtime:/etc/localtime:ro
command:
- --addr=0.0.0.0:20160
- --advertise-addr=127.0.0.1:20160
- --data-dir=/data/tikv0
- --pd=127.0.0.1:2379
- --log-file=/logs/tikv0.log
depends_on:
- "pd0"
restart: on-failure

85
docker-compose.yml
View File

@ -1,60 +1,31 @@
version: '3.5'
x-ccache: &ccache
CCACHE_COMPILERCHECK: content
CCACHE_COMPRESS: 1
CCACHE_COMPRESSLEVEL: 5
CCACHE_MAXSIZE: 2G
CCACHE_DIR: /ccache
services:
etcd:
image: quay.io/coreos/etcd:latest
command: etcd -listen-peer-urls=http://127.0.0.1:12380 -advertise-client-urls=http://127.0.0.1:12379 -listen-client-urls http://0.0.0.0:12379,http://0.0.0.0:14001 -initial-advertise-peer-urls=http://127.0.0.1:12380 --initial-cluster default=http://127.0.0.1:12380
ports:
- "12379:12379"
- "12380:12380"
- "14001:14001"
pulsar:
image: apachepulsar/pulsar:latest
command: bin/pulsar standalone
ports:
- "6650:6650"
- "18080:8080"
pd0:
image: pingcap/pd:latest
network_mode: "host"
ports:
- "2379:2379"
- "2380:2380"
volumes:
- /tmp/config/pd.toml:/pd.toml:ro
- /tmp/data:/data
- /tmp/logs:/logs
- /etc/localtime:/etc/localtime:ro
command:
- --name=pd0
- --client-urls=http://0.0.0.0:2379
- --peer-urls=http://0.0.0.0:2380
- --advertise-client-urls=http://127.0.0.1:2379
- --advertise-peer-urls=http://127.0.0.1:2380
- --initial-cluster=pd0=http://127.0.0.1:2380
- --data-dir=/data/pd0
- --log-file=/logs/pd0.log
restart: on-failure
tikv0:
network_mode: "host"
image: pingcap/tikv:latest
ports:
- "20160:20160"
volumes:
- /tmp/config/tikv.toml:/tikv.toml:ro
- /tmp/data:/data
- /tmp/logs:/logs
- /etc/localtime:/etc/localtime:ro
command:
- --addr=0.0.0.0:20160
- --advertise-addr=127.0.0.1:20160
- --data-dir=/data/tikv0
- --pd=127.0.0.1:2379
- --log-file=/logs/tikv0.log
depends_on:
- "pd0"
restart: on-failure
ubuntu:
image: ${REPO}:${ARCH}-ubuntu${UBUNTU}
build:
context: .
dockerfile: build/docker/env/cpu/ubuntu${UBUNTU}/Dockerfile
cache_from:
- ${REPO}:${ARCH}-ubuntu${UBUNTU}
shm_size: 2G
environment:
<<: *ccache
volumes: &ubuntu-volumes
- .:/milvus-distributed:delegated
- ${DOCKER_VOLUME_DIRECTORY:-.docker}/${ARCH}-ubuntu${UBUNTU}-cache:/ccache:delegated
working_dir: "/milvus-distributed"
command: &ubuntu-command >
/bin/bash -c "
/milvus-distributed/scripts/core_build.sh -u && \
go build -o /milvus-distributed/cmd/writer/writer /milvus-distributed/cmd/writer/writer.go && \
go build -o /milvus-distributed/cmd/reader/reader /milvus-distributed/cmd/reader/reader.go && \
go build -o /milvus-distributed/cmd/master/master /milvus-distributed/cmd/master/master.go && \
go build -o /milvus-distributed/cmd/proxy/proxy /milvus-distributed/cmd/proxy/proxy.go"

19
docs/developer_guides/developer_guides.md
View File

@ -442,17 +442,17 @@ type TsMsg interface {
Ts() Timestamp
}
type TsMsgMarshaler interface {
Marshal(input *TsMsg) ([]byte, Status)
Unmarshal(input []byte) (*TsMsg, Status)
}
type MsgPack struct {
BeginTs Timestamp
EndTs Timestamp
Msgs []*TsMsg
}
type TsMsgMarshaler interface {
Marshal(input *TsMsg) ([]byte, Status)
Unmarshal(input []byte) (*TsMsg, Status)
}
type MsgStream interface {
SetMsgMarshaler(marshal *TsMsgMarshaler, unmarshal *TsMsgMarshaler)
Produce(*MsgPack) Status
@ -461,14 +461,17 @@ type MsgStream interface {
type PulsarMsgStream struct {
client *pulsar.Client
produceChannels []string
consumeChannels []string
msgHashFunc (*MsgPack) map[int32]*MsgPack // return a map from produceChannel idx to *MsgPack
producers []*pulsar.Producer
consumers []*pulsar.Consumer
msgMarshaler *TsMsgMarshaler
msgUnmarshaler *TsMsgMarshaler
}
func (ms *PulsarMsgStream) SetProducerChannels(channels []string)
func (ms *PulsarMsgStream) SetConsumerChannels(channels []string)
func (ms *PulsarMsgStream) SetMsgMarshaler(marshal *TsMsgMarshaler, unmarshal *TsMsgMarshaler)
func (ms *PulsarMsgStream) SetMsgHashFunc(XXX)
func (ms *PulsarMsgStream) Produce(*MsgPack) Status
func (ms *PulsarMsgStream) Consume() *MsgPack //return messages in one time tick

11
internal/core/build.sh
View File

@ -5,10 +5,19 @@ if [[ ! ${jobs+1} ]]; then
jobs=$(nproc)
fi
SOURCE="${BASH_SOURCE[0]}"
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
SOURCE="$(readlink "$SOURCE")"
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
done
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
BUILD_OUTPUT_DIR="cmake_build"
BUILD_TYPE="Release"
BUILD_UNITTEST="OFF"
INSTALL_PREFIX=$(pwd)/output
INSTALL_PREFIX=${SCRIPTS_DIR}/output
MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/tmp/milvus"

58
internal/core/cmake/FindGTest.cmake
View File

@ -1,58 +0,0 @@
find_package(Threads REQUIRED)
include(ExternalProject)
ExternalProject_Add(
googletest
URL http://ss2.fluorinedog.com/data/gtest_v1.10.x.zip
UPDATE_COMMAND ""
INSTALL_COMMAND ""
LOG_DOWNLOAD ON
LOG_CONFIGURE ON
LOG_BUILD ON)
ExternalProject_Get_Property(googletest source_dir)
set(GTEST_INCLUDE_DIRS ${source_dir}/googletest/include)
set(GMOCK_INCLUDE_DIRS ${source_dir}/googlemock/include)
# The cloning of the above repo doesn't happen until make, however if the dir doesn't
# exist, INTERFACE_INCLUDE_DIRECTORIES will throw an error.
# To make it work, we just create the directory now during config.
file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS})
file(MAKE_DIRECTORY ${GMOCK_INCLUDE_DIRS})
ExternalProject_Get_Property(googletest binary_dir)
set(GTEST_LIBRARY_PATH ${binary_dir}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest.a)
set(GTEST_LIBRARY gtest)
add_library(${GTEST_LIBRARY} UNKNOWN IMPORTED)
set_target_properties(${GTEST_LIBRARY} PROPERTIES
"IMPORTED_LOCATION" "${GTEST_LIBRARY_PATH}"
"IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}"
"INTERFACE_INCLUDE_DIRECTORIES" "${GTEST_INCLUDE_DIRS}")
add_dependencies(${GTEST_LIBRARY} googletest)
set(GTEST_MAIN_LIBRARY_PATH ${binary_dir}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest_main.a)
set(GTEST_MAIN_LIBRARY gtest_main)
add_library(${GTEST_MAIN_LIBRARY} UNKNOWN IMPORTED)
set_target_properties(${GTEST_MAIN_LIBRARY} PROPERTIES
"IMPORTED_LOCATION" "${GTEST_MAIN_LIBRARY_PATH}"
"IMPORTED_LINK_INTERFACE_LIBRARImS" "${CMAKE_THREAD_LIBS_INIT}"
"INTERFACE_INCLUDE_DIRECTORIES" "${GTEST_INCLUDE_DIRS}")
add_dependencies(${GTEST_MAIN_LIBRARY} googletest)
# set(GMOCK_LIBRARY_PATH ${binary_dir}/googlemock/${CMAKE_FIND_LIBRARY_PREFIXES}gmock.a)
# set(GMOCK_LIBRARY gmock)
# add_library(${GMOCK_LIBRARY} UNKNOWN IMPORTED)
# set_target_properties(${GMOCK_LIBRARY} PROPERTIES
# "IMPORTED_LOCATION" "${GMOCK_LIBRARY_PATH}"
# "IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}"
# "INTERFACE_INCLUDE_DIRECTORIES" "${GMOCK_INCLUDE_DIRS}")
# add_dependencies(${GMOCK_LIBRARY} googletest)
# set(GMOCK_MAIN_LIBRARY_PATH ${binary_dir}/googlemock/${CMAKE_FIND_LIBRARY_PREFIXES}gmock_main.a)
# set(GMOCK_MAIN_LIBRARY gmock_main)
# add_library(${GMOCK_MAIN_LIBRARY} UNKNOWN IMPORTED)
# set_target_properties(${GMOCK_MAIN_LIBRARY} PROPERTIES
# "IMPORTED_LOCATION" "${GMOCK_MAIN_LIBRARY_PATH}"
# "IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}"
# "INTERFACE_INCLUDE_DIRECTORIES" "${GMOCK_INCLUDE_DIRS}")
# add_dependencies(${GMOCK_MAIN_LIBRARY} ${GTEST_LIBRARY})

44
internal/core/src/CMakeLists.txt
View File

@ -29,47 +29,3 @@ add_subdirectory( log)
add_subdirectory( dog_segment)
add_subdirectory( cache )
add_subdirectory( query )
# add_subdirectory( db ) # target milvus_engine
# add_subdirectory( server )
# set(link_lib
# milvus_engine
# # dog_segment
# #query
# utils
# curl
# )
# set( BOOST_LIB libboost_system.a
# libboost_filesystem.a
# libboost_serialization.a
# )
# set( THIRD_PARTY_LIBS yaml-cpp
# )
# target_link_libraries( server
# PUBLIC ${link_lib}
# ${THIRD_PARTY_LIBS}
# ${BOOST_LIB}
# )
# # **************************** Get&Print Include Directories ****************************
# get_property( dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES )
# foreach ( dir ${dirs} )
# message( STATUS "Current Include DIRS: ")
# endforeach ()
# set( SERVER_LIBS server )
# add_executable( milvus_server ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp
# )
# #target_include_directories(db PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar-client-cpp/pulsar-client-cpp-src/pulsar-client-cpp/include)
# target_link_libraries( milvus_server PRIVATE ${SERVER_LIBS} )
# install( TARGETS milvus_server DESTINATION bin )

94
internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake
View File

@ -14,7 +14,6 @@
set(KNOWHERE_THIRDPARTY_DEPENDENCIES
Arrow
FAISS
GTest
OpenBLAS
MKL
)
@ -31,8 +30,6 @@ endforeach ()
macro(build_dependency DEPENDENCY_NAME)
if ("${DEPENDENCY_NAME}" STREQUAL "Arrow")
build_arrow()
elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest")
find_package(GTest REQUIRED)
elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS")
build_openblas()
elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS")
@ -215,12 +212,6 @@ else ()
)
endif ()
# if (DEFINED ENV{KNOWHERE_GTEST_URL})
# set(GTEST_SOURCE_URL "$ENV{KNOWHERE_GTEST_URL}")
# else ()
# set(GTEST_SOURCE_URL
# "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz")
# endif ()
if (DEFINED ENV{KNOWHERE_OPENBLAS_URL})
set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}")
@ -383,91 +374,6 @@ if (KNOWHERE_WITH_OPENBLAS)
link_directories(SYSTEM "${OpenBLAS_LIB_DIR}")
endif()
# ----------------------------------------------------------------------
# Google gtest
# macro(build_gtest)
# message(STATUS "Building gtest-${GTEST_VERSION} from source")
# set(GTEST_VENDORED TRUE)
# set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}")
#
# if (APPLE)
# set(GTEST_CMAKE_CXX_FLAGS
# ${GTEST_CMAKE_CXX_FLAGS}
# -DGTEST_USE_OWN_TR1_TUPLE=1
# -Wno-unused-value
# -Wno-ignored-attributes)
# endif ()
#
# set(GTEST_PREFIX "${INDEX_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep")
# set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include")
# set(GTEST_STATIC_LIB
# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}")
# set(GTEST_MAIN_STATIC_LIB
# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}")
#
# set(GTEST_CMAKE_ARGS
# ${EP_COMMON_CMAKE_ARGS}
# "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}"
# "-DCMAKE_INSTALL_LIBDIR=lib"
# -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS}
# -DCMAKE_BUILD_TYPE=Release)
#
# set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include")
# set(GMOCK_STATIC_LIB
# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}"
# )
#
# ExternalProject_Add(googletest_ep
# URL
# ${GTEST_SOURCE_URL}
# BUILD_COMMAND
# ${MAKE}
# ${MAKE_BUILD_ARGS}
# BUILD_BYPRODUCTS
# ${GTEST_STATIC_LIB}
# ${GTEST_MAIN_STATIC_LIB}
# ${GMOCK_STATIC_LIB}
# CMAKE_ARGS
# ${GTEST_CMAKE_ARGS}
# ${EP_LOG_OPTIONS})
#
# # The include directory must exist before it is referenced by a target.
# file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}")
#
# add_library(gtest STATIC IMPORTED)
# set_target_properties(gtest
# PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}"
# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
#
# add_library(gtest_main STATIC IMPORTED)
# set_target_properties(gtest_main
# PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}"
# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
#
# add_library(gmock STATIC IMPORTED)
# set_target_properties(gmock
# PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}"
# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}")
#
# add_dependencies(gtest googletest_ep)
# add_dependencies(gtest_main googletest_ep)
# add_dependencies(gmock googletest_ep)
#
# endmacro()
## if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep)
#if ( NOT TARGET gtest AND KNOWHERE_BUILD_TESTS )
# resolve_dependency(GTest)
#
# if (NOT GTEST_VENDORED)
# endif ()
#
# # TODO: Don't use global includes but rather target_include_directories
# get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES)
# link_directories(SYSTEM "${GTEST_PREFIX}/lib")
# include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
#endif ()
# ----------------------------------------------------------------------
# MKL

50
internal/core/src/main.cpp
View File

@ -1,50 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <getopt.h>
#include <unistd.h>
#include <csignal>
#include <cstring>
#include <string>
#include "config/ConfigMgr.h"
#include "easyloggingpp/easylogging++.h"
#include "utils/SignalHandler.h"
#include "utils/Status.h"
INITIALIZE_EASYLOGGINGPP
void
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl;
std::cout << R"(
Options:
-h --help Print this help.
-c --conf_file filename Read configuration from the file.
-d --daemon Daemonize this application.
-p --pid_file filename PID file used by daemonized app.
)" << std::endl;
}
void
print_banner() {
std::cout << std::endl;
std::cout << " __ _________ _ ____ ______ " << std::endl;
std::cout << " / |/ / _/ /| | / / / / / __/ " << std::endl;
std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl;
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
std::cout << std::endl;
}
int
main(int argc, char* argv[]) {
print_banner();
}

6
internal/core/thirdparty/CMakeLists.txt vendored
View File

@ -32,6 +32,12 @@ set( FETCHCONTENT_QUIET OFF )
set( THREADS_PREFER_PTHREAD_FLAG ON )
find_package( Threads REQUIRED )
# ****************************** Thirdparty googletest ***************************************
if ( MILVUS_BUILD_TESTS )
add_subdirectory( gtest )
endif()
# ****************************** Thirdparty yaml ***************************************
if ( MILVUS_WITH_YAMLCPP )
add_subdirectory( yaml-cpp )

64
internal/core/thirdparty/gtest/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,64 @@
#-------------------------------------------------------------------------------
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
# or implied. See the License for the specific language governing permissions and limitations under the License.
#-------------------------------------------------------------------------------
if ( DEFINED ENV{MILVUS_GTEST_URL} )
set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" )
else()
set( GTEST_SOURCE_URL "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" )
endif()
message( STATUS "Building gtest-${GTEST_VERSION} from source" )
set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only
FetchContent_Declare(
googletest
URL ${GTEST_SOURCE_URL}
URL_MD5 "f9137c5bc18b7d74027936f0f1bfa5c8"
SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build
DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} )
FetchContent_GetProperties( googletest )
if ( NOT googletest_POPULATED )
FetchContent_Populate( googletest )
# Adding the following targets:
# gtest, gtest_main, gmock, gmock_main
add_subdirectory( ${googletest_SOURCE_DIR}
${googletest_BINARY_DIR}
EXCLUDE_FROM_ALL )
endif()
# ****************************************************************
# Create ALIAS Target
# ****************************************************************
# if (NOT TARGET GTest:gtest)
# add_library( GTest::gtest ALIAS gtest )
# endif()
# if (NOT TARGET GTest:main)
# add_library( GTest::main ALIAS gtest_main )
# endif()
# if (NOT TARGET GMock:gmock)
# target_link_libraries( gmock INTERFACE GTest::gtest )
# add_library( GMock::gmock ALIAS gmock )
# endif()
# if (NOT TARGET GMock:main)
# target_link_libraries( gmock_main INTERFACE GTest::gtest )
# add_library( GMock::main ALIAS gmock_main )
# endif()
get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS )
message( STATUS "gtest compile options: ${var}" )

1
internal/core/thirdparty/protobuf/CMakeLists.txt vendored
View File

@ -31,6 +31,7 @@ FetchContent_Declare(
)
set( protobuf_BUILD_TESTS CACHE BOOL OFF FORCE )
if ( NOT protobuf_POPULATED )
FetchContent_Populate( protobuf )

5
internal/core/unittest/CMakeLists.txt
View File

@ -1,8 +1,7 @@
enable_testing()
find_package(GTest REQUIRED)
include_directories(${CMAKE_HOME_DIRECTORY}/src)
include_directories(>>>> ${CMAKE_HOME_DIRECTORY}/src/index/knowhere)
include_directories(${CMAKE_HOME_DIRECTORY}/src/index/knowhere)
set(MILVUS_TEST_FILES
test_naive.cpp
test_dog_segment.cpp
@ -22,4 +21,4 @@ target_link_libraries(all_tests
log
pthread
)
install (TARGETS all_tests DESTINATION unittest)
install (TARGETS all_tests DESTINATION unittest)

32
internal/proxy/manipulation_req.go
View File

@ -12,7 +12,7 @@ import (
)
type manipulationReq struct {
stats []commonpb.Status
commonpb.Status
msgs []*pb.ManipulationReqMsg
wg sync.WaitGroup
proxy *proxyServer
@ -26,13 +26,14 @@ func (req *manipulationReq) Ts() (Timestamp, error) {
return Timestamp(req.msgs[0].Timestamp), nil
}
func (req *manipulationReq) SetTs(ts Timestamp) {
for _, msg := range req.msgs {
msg.Timestamp = uint64(ts)
for _, mreq := range req.msgs {
mreq.Timestamp = uint64(ts)
}
}
// BaseRequest interfaces
func (req *manipulationReq) Type() pb.ReqType {
// TODO: return a invalid ReqType?
if req.msgs == nil {
return 0
}
@ -58,17 +59,11 @@ func (req *manipulationReq) Execute() commonpb.Status {
func (req *manipulationReq) PostExecute() commonpb.Status { // send into pulsar
req.wg.Add(1)
return commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS}
return req.Status
}
func (req *manipulationReq) WaitToFinish() commonpb.Status { // wait until send into pulsar
req.wg.Wait()
for _, stat := range req.stats{
if stat.ErrorCode != commonpb.ErrorCode_SUCCESS{
return stat
}
}
// update timestamp if necessary
ts, _ := req.Ts()
req.proxy.reqSch.mTimestampMux.Lock()
@ -78,7 +73,7 @@ func (req *manipulationReq) WaitToFinish() commonpb.Status { // wait until send
} else {
log.Printf("there is some wrong with m_timestamp, it goes back, current = %d, previous = %d", ts, req.proxy.reqSch.mTimestamp)
}
return req.stats[0]
return req.Status
}
func (s *proxyServer) restartManipulationRoutine(bufSize int) error {
@ -114,23 +109,22 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error {
ts, st := s.getTimestamp(1)
if st.ErrorCode != commonpb.ErrorCode_SUCCESS {
log.Printf("get time stamp failed, error code = %d, msg = %s", st.ErrorCode, st.Reason)
ip.stats[0] = st
ip.Status = st
ip.wg.Done()
break
}
ip.SetTs(ts[0])
wg := sync.WaitGroup{}
for i, mq := range ip.msgs {
for _, mq := range ip.msgs {
mq := mq
i := i
wg.Add(1)
go func() {
wg.Add(1)
defer wg.Done()
mb, err := proto.Marshal(mq)
if err != nil {
log.Printf("Marshal ManipulationReqMsg failed, error = %v", err)
ip.stats[i] = commonpb.Status{
ip.Status = commonpb.Status{
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
Reason: fmt.Sprintf("Marshal ManipulationReqMsg failed, error=%v", err),
}
@ -141,7 +135,7 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error {
case pb.ReqType_kInsert:
if _, err := readers[mq.ChannelId].Send(s.ctx, &pulsar.ProducerMessage{Payload: mb}); err != nil {
log.Printf("post into puslar failed, error = %v", err)
ip.stats[i] = commonpb.Status{
ip.Status = commonpb.Status{
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
Reason: fmt.Sprintf("Post into puslar failed, error=%v", err.Error()),
}
@ -150,10 +144,6 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error {
case pb.ReqType_kDeleteEntityByID:
if _, err = deleter.Send(s.ctx, &pulsar.ProducerMessage{Payload: mb}); err != nil {
log.Printf("post into pulsar filed, error = %v", err)
ip.stats[i] = commonpb.Status{
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
Reason: fmt.Sprintf("Post into puslar failed, error=%v", err.Error()),
}
return
}
default:

9
internal/proxy/server.go
View File

@ -156,7 +156,6 @@ func (s *proxyServer) DeleteByID(ctx context.Context, req *pb.DeleteByIDParam) (
}
if len(mReqMsg.PrimaryKeys) > 1 {
mReq := &manipulationReq{
stats: make([]commonpb.Status, 1),
msgs: append([]*pb.ManipulationReqMsg{}, &mReqMsg),
proxy: s,
}
@ -223,10 +222,10 @@ func (s *proxyServer) Insert(ctx context.Context, req *servicepb.RowBatch) (*ser
// TODO: alloc manipulation request id
mReq := manipulationReq{
stats: make([]commonpb.Status, len(msgMap)),
msgs: make([]*pb.ManipulationReqMsg, len(msgMap)),
wg: sync.WaitGroup{},
proxy: s,
Status: commonpb.Status{},
msgs: make([]*pb.ManipulationReqMsg, len(msgMap)),
wg: sync.WaitGroup{},
proxy: s,
}
for _, v := range msgMap {
mReq.msgs = append(mReq.msgs, v)

2
scripts/core_build.sh
View File

@ -19,7 +19,7 @@ CPP_BUILD_DIR="${CPP_SRC_DIR}/cmake_build"
BUILD_OUTPUT_DIR=${CPP_BUILD_DIR}
BUILD_TYPE="Release"
BUILD_UNITTEST="OFF"
INSTALL_PREFIX="${CPP_SRC_DIR}/milvus"
INSTALL_PREFIX="${CPP_SRC_DIR}/output"
MAKE_CLEAN="OFF"
BUILD_COVERAGE="OFF"
DB_PATH="/tmp/milvus"