Update compile options and config for embedded Milvus (#16472)

/kind enhancement

issue: #15711
Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>

Makefile

@@ -136,15 +136,15 @@ build-go: milvus
 
 build-cpp: pre-proc
 	@echo "Building Milvus cpp library ..."
-	@(env bash $(PWD)/scripts/core_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
+	@(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
 	@(env bash $(PWD)/scripts/cwrapper_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
 	@(env bash $(PWD)/scripts/cwrapper_rocksdb_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
 
 build-cpp-embd: pre-proc
 	@echo "Building **Embedded** Milvus cpp library ..."
 	@(env bash $(PWD)/scripts/core_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
-	@(env bash $(PWD)/scripts/cwrapper_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
-	@(env bash $(PWD)/scripts/cwrapper_rocksdb_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
+	@(env bash $(PWD)/scripts/cwrapper_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
+	@(env bash $(PWD)/scripts/cwrapper_rocksdb_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)")
 
 build-cpp-with-unittest: pre-proc
 	@echo "Building Milvus cpp library with unittest ..."

configs/embedded-milvus.yaml

@@ -27,11 +27,19 @@ etcd:
     #  - "stderr" as os.Stderr,
     #  - "stdout" as os.Stdout,
     #  - file path to append server logs to.
+    # (adjusted for embedded Milvus)
     path: /tmp/milvus/logs/etcd.log
+    # (adjusted for embedded Milvus)
     level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
   use:
+    # (adjusted for embedded Milvus)
     embed: true # Whether to enable embedded Etcd (an in-process EtcdServer).
 
+# (adjusted for embedded Milvus)
+localStorage:
+  path: /tmp/milvus/data/
+  enabled: true
+
 # Related configuration of minio, which is responsible for data persistence for Milvus.
 minio:
   address: localhost # Address of MinIO/S3
@@ -49,6 +57,7 @@ pulsar:
   maxMessageSize: 5242880 # 5 * 1024 * 1024 Bytes, Maximum size of each message in pulsar.
 
 rocksmq:
+  # (adjusted for embedded Milvus)
   path: /tmp/milvus/rdb_data # The path where the message is stored in rocksmq
   rocksmqPageSize: 2147483648 # 2 GB, 2 * 1024 * 1024 * 1024 bytes, The size of each page of messages in rocksmq
   retentionTimeInMinutes: 10080 # 7 days, 7 * 24 * 60 minutes, The retention time of the message in rocksmq.
@@ -69,6 +78,22 @@ rootCoord:
   maxPartitionNum: 4096 # Maximum number of partitions in a collection
   minSegmentSizeToEnableIndex: 1024 # It's a threshold. When the segment size is less than this value, the segment will not be indexed
 
+  # (in seconds) Duration after which an import task will expire (be killed). Default 3600 seconds (1 hour).
+  # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
+  importTaskExpiration: 3600
+  # (in seconds) Milvus will keep the record of import tasks for at least `importTaskRetention` seconds. Default 86400
+  # seconds (24 hours).
+  # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
+  importTaskRetention: 86400
+  # (in seconds) During index building phase of an import task, Milvus will check the building status of a task's
+  # segments' indices every `importIndexCheckInterval` seconds. Default 300 seconds (5 minutes).
+  # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
+  importIndexCheckInterval: 300
+  # (in seconds) Maximum time to wait before pushing flushed segments online (make them searchable) during importing.
+  # Default 1200 seconds (20 minutes).
+  # Note: If default value is to be changed, change also the default in: internal/util/paramtable/component_param.go
+  importIndexWaitLimit: 1200
+
 # Related configuration of proxy, used to validate client requests and reduce the returned results.
 proxy:
   port: 19530
@@ -94,8 +119,7 @@ proxy:
   maxDimension: 32768 # Maximum dimension of a vector
   maxShardNum: 256 # Maximum number of shards in a collection
   maxTaskNum: 1024 # max task number of proxy task queue
-  bufFlagExpireTime: 300 # second, the time to expire bufFlag from cache in collectResultLoop. Default 300.
-  bufFlagCleanupInterval: 600 # second, the interval to clean bufFlag cache in collectResultLoop. Default 600.
+  # (adjusted for embedded Milvus)
   ginLogging: false # Whether to produce gin logs.
 
 
@@ -126,7 +150,6 @@ queryNode:
     serverMaxSendSize: 2147483647 # math.MaxInt32
     clientMaxRecvSize: 104857600 # 100 MB, 100 * 1024 * 1024
     clientMaxSendSize: 104857600 # 100 MB, 100 * 1024 * 1024
-
   stats:
     publishInterval: 1000 # Interval for querynode to report node information (milliseconds)
   dataSync:
@@ -142,6 +165,9 @@ queryNode:
   # Segcore will divide a segment into multiple chunks.
   segcore:
     chunkRows: 32768 # The number of vectors in a chunk.
+  cache:
+    enabled: true
+    memoryLimit: 2147483648 # 2 GB, 2 * 1024 *1024 *1024
 
 
 indexCoord:
@@ -207,15 +233,12 @@ dataNode:
     # Max buffer size to flush for a single segment.
     insertBufSize: 16777216 # Bytes, 16 MB
 
-# Configure whether to store the vector and the local path when querying/searching in Querynode.
-localStorage:
-  path: /tmp/milvus/data/
-  enabled: true
-
 # Configures the system log output.
 log:
+  # (adjusted for embedded Milvus)
   level: info # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
   file:
+    # (adjusted for embedded Milvus)
     rootPath: /tmp/milvus/logs # default to stdout, stderr
     maxSize: 300 # MB
     maxAge: 10 # Maximum time for log retention in day.
@@ -256,3 +279,7 @@ common:
   # Valid values: [auto, avx512, avx2, avx, sse4_2]
   # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building.
   simdType: auto
+  indexSliceSize: 4 # MB
+
+  storage:
+    vector: minio

internal/distributed/proxy/service.go

@@ -378,6 +378,8 @@ func (s *Server) init() error {
 		return err
 	}
 	log.Debug("init Proxy done")
+	// Intentionally print to stdout, which is usually a sign that Milvus is ready to serve.
+	fmt.Println("---Milvus Proxy successfully initialized and ready to serve!---")
 
 	return nil
 }

internal/kv/rocksdb/cwrapper/CMakeLists.txt

@@ -56,11 +56,16 @@ macro( build_rocksdb )
         set ( ROCKSDB_URL_MD5 "67f9e04fda62af551dd039c37b1808ac" )
     endif ()
 
+    if ( EMBEDDED_MILVUS )
+        message ( STATUS "Turning on fPIC while building embedded Milvus" )
+        set( FPIC_ARG "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" )
+    endif()
     set( ROCKSDB_CMAKE_ARGS
          "-DWITH_GFLAGS=OFF"
          "-DROCKSDB_BUILD_SHARED=OFF"
          "-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
          "-DFAIL_ON_WARNINGS=OFF"
+         ${FPIC_ARG}
          #This is used to solve 'illegal instruction' problem in some machine
          "-DPORTABLE=ON"
         )

internal/storage/cwrapper/CMakeLists.txt

@@ -126,7 +126,14 @@ endif (MSYS)
 
 add_library(wrapper STATIC)
 target_sources(wrapper PUBLIC ParquetWrapper.cpp PayloadStream.cpp)
-set_target_properties( wrapper PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} )
+
+if ( EMBEDDED_MILVUS )
+    message ( STATUS "Turning on fPIC while building embedded Milvus" )
+    set_target_properties( wrapper PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} POSITION_INDEPENDENT_CODE ON )
+else()
+    set_target_properties( wrapper PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_CURRENT_SOURCE_DIR} )
+endif()
+
 target_link_libraries( wrapper PUBLIC parquet pthread)
 
 if(NOT CMAKE_INSTALL_PREFIX)

scripts/cwrapper_build.sh

@@ -37,8 +37,9 @@ mkdir -p ${OUTPUT_LIB}
 
 BUILD_TYPE="Debug"
 CUSTOM_THIRDPARTY_PATH=""
+EMBEDDED_MILVUS="OFF"
 
-while getopts "a:b:t:h:f:" arg; do
+while getopts "a:g:t:h:f:b" arg; do
   case $arg in
   f)
     CUSTOM_THIRDPARTY_PATH=$OPTARG
@@ -49,18 +50,22 @@ while getopts "a:b:t:h:f:" arg; do
   a)
     GIT_ARROW_REPO=$OPTARG
     ;;
-  b)
+  g)
     GIT_ARROW_TAG=$OPTARG
     ;;
   h) # help
     echo "-t: build type(default: Debug)
 -a: arrow repo(default: https://github.com/apache/arrow.git)
--b: arrow tag(default: apache-arrow-2.0.0)
+-g: arrow tag(default: apache-arrow-2.0.0)
 -f: custom thirdparty path(default: "")
 -h: help
+-b: build embedded milvus(default: OFF)
                 "
     exit 0
     ;;
+  b)
+    EMBEDDED_MILVUS="ON"
+    ;;
   ?)
     echo "ERROR! unknown argument"
     exit 1
@@ -82,6 +87,7 @@ CMAKE_CMD="cmake \
 ${CMAKE_EXTRA_ARGS} \
 -DCMAKE_INSTALL_PREFIX=${OUTPUT_LIB} \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DEMBEDDED_MILVUS=${EMBEDDED_MILVUS} \
 -DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} ${SRC_DIR}"
 
 ${CMAKE_CMD} -G "${CMAKE_GENERATOR}"

scripts/cwrapper_rocksdb_build.sh

@@ -39,8 +39,9 @@ fi
 
 BUILD_TYPE="Debug"
 CUSTOM_THIRDPARTY_PATH=""
+EMBEDDED_MILVUS="OFF"
 
-while getopts "t:h:f:" arg; do
+while getopts "t:h:f:b" arg; do
   case $arg in
   f)
     CUSTOM_THIRDPARTY_PATH=$OPTARG
@@ -52,9 +53,13 @@ while getopts "t:h:f:" arg; do
     echo "-t: build type(default: Debug)
 -f: custom thirdparty path(default: "")
 -h: help
+-b: build embedded milvus(default: OFF)
                 "
     exit 0
     ;;
+  b)
+    EMBEDDED_MILVUS="ON"
+    ;;
   ?)
     echo "ERROR! unknown argument"
     exit 1
@@ -68,6 +73,7 @@ pushd ${CMAKE_BUILD}
 CMAKE_CMD="cmake \
 -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
 -DCMAKE_INSTALL_PREFIX=${OUTPUT_LIB} \
+-DEMBEDDED_MILVUS=${EMBEDDED_MILVUS} \
 -DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} ${SRC_DIR}"
 
 unameOut="$(uname -s)"