diff --git a/.jenkins/modules/Build/Build.groovy b/.jenkins/modules/Build/Build.groovy index 14fd0b9cdd..1d35fd71ff 100644 --- a/.jenkins/modules/Build/Build.groovy +++ b/.jenkins/modules/Build/Build.groovy @@ -1,22 +1,20 @@ timeout(time: 20, unit: 'MINUTES') { - dir ("scripts") { - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz || echo \"Go mod artfactory files not found!\"' - } + + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz || echo \"Go mod artfactory files not found!\"' // Zero the cache statistics (but not the configuration options) sh 'ccache -z' sh '. ./scripts/before-install.sh && make install' sh 'echo -e "===\n=== ccache statistics after build\n===" && ccache --show-stats' - dir ("scripts") { - withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./before-install.sh && unset http_proxy && unset https_proxy && ./update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-cache.tar.gz -u ${USERNAME} -p ${PASSWORD}' - } + + withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}' + sh '. ./scripts/before-install.sh && unset http_proxy && unset https_proxy && ./scripts/update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-distributed-go-mod-\$(md5sum go.mod).tar.gz -u ${USERNAME} -p ${PASSWORD}' } } diff --git a/internal/core/src/indexbuilder/IndexWrapper.cpp b/internal/core/src/indexbuilder/IndexWrapper.cpp index 4484d1e119..1a26404aa2 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.cpp +++ b/internal/core/src/indexbuilder/IndexWrapper.cpp @@ -34,6 +34,9 @@ IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* seria auto mode = get_config_by_name("index_mode"); auto index_mode = mode.has_value() ? mode_map[mode.value()] : knowhere::IndexMode::MODE_CPU; + auto index_type = get_index_type(); + auto metric_type = get_metric_type(); + AssertInfo(!is_unsupported(index_type, metric_type), index_type + " doesn't support metric: " + metric_type); index_ = knowhere::VecIndexFactory::GetInstance().CreateVecIndex(get_index_type(), index_mode); Assert(index_ != nullptr); } @@ -263,6 +266,21 @@ IndexWrapper::get_index_type() { return type.has_value() ? type.value() : knowhere::IndexEnum::INDEX_FAISS_IVFPQ; } +std::string +IndexWrapper::get_metric_type() { + auto type = get_config_by_name(knowhere::Metric::TYPE); + if (type.has_value()) { + return type.value(); + } else { + auto index_type = get_index_type(); + if (is_in_bin_list(index_type)) { + return knowhere::Metric::JACCARD; + } else { + return knowhere::Metric::L2; + } + } +} + std::unique_ptr IndexWrapper::Query(const knowhere::DatasetPtr& dataset) { return std::move(QueryImpl(dataset, config_)); diff --git a/internal/core/src/indexbuilder/IndexWrapper.h b/internal/core/src/indexbuilder/IndexWrapper.h index 16f2721712..979a8fa960 100644 --- a/internal/core/src/indexbuilder/IndexWrapper.h +++ b/internal/core/src/indexbuilder/IndexWrapper.h @@ -59,6 +59,9 @@ class IndexWrapper { std::string get_index_type(); + std::string + get_metric_type(); + template std::optional get_config_by_name(std::string name); diff --git a/internal/core/src/indexbuilder/utils.h b/internal/core/src/indexbuilder/utils.h index e1ed080496..6e1d89a2d9 100644 --- a/internal/core/src/indexbuilder/utils.h +++ b/internal/core/src/indexbuilder/utils.h @@ -14,6 +14,7 @@ #include #include #include +#include #include "index/knowhere/knowhere/index/IndexType.h" @@ -57,6 +58,14 @@ Need_BuildAll_list() { return ret; } +std::vector> +unsupported_index_combinations() { + static std::vector> ret{ + std::make_tuple(std::string(knowhere::IndexEnum::INDEX_FAISS_BIN_IVFFLAT), std::string(knowhere::Metric::L2)), + }; + return ret; +} + template bool is_in_list(const T& t, std::function()> list_func) { @@ -84,5 +93,11 @@ is_in_need_id_list(const milvus::knowhere::IndexType& index_type) { return is_in_list(index_type, Need_ID_List); } +bool +is_unsupported(const milvus::knowhere::IndexType& index_type, const milvus::knowhere::MetricType& metric_type) { + return is_in_list>(std::make_tuple(index_type, metric_type), + unsupported_index_combinations); +} + } // namespace indexbuilder } // namespace milvus diff --git a/internal/indexbuilder/index.go b/internal/indexbuilder/index.go index fceec4bcf0..eba049e543 100644 --- a/internal/indexbuilder/index.go +++ b/internal/indexbuilder/index.go @@ -14,6 +14,7 @@ package indexbuilder import "C" import ( "errors" + "fmt" "strconv" "unsafe" @@ -142,6 +143,8 @@ func (index *CIndex) Delete() error { } func NewCIndex(typeParams, indexParams map[string]string) (Index, error) { + fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN typeParams: ", typeParams) + fmt.Println("NNNNNNNNNNNNNNNNNNNNNNNNNNN indexParams: ", indexParams) protoTypeParams := &indexcgopb.TypeParams{ Params: make([]*commonpb.KeyValuePair, 0), } @@ -168,10 +171,14 @@ func NewCIndex(typeParams, indexParams map[string]string) (Index, error) { CIndex* res_index); */ var indexPtr C.CIndex + fmt.Println("before create index ........................................") status := C.CreateIndex(typeParamsPointer, indexParamsPointer, &indexPtr) + fmt.Println("after create index ........................................") errorCode := status.error_code + fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error code: ", errorCode) if errorCode != 0 { errorMsg := C.GoString(status.error_msg) + fmt.Println("EEEEEEEEEEEEEEEEEEEEEEEEEE error msg: ", errorMsg) defer C.free(unsafe.Pointer(status.error_msg)) return nil, errors.New(" failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg) } diff --git a/internal/indexbuilder/task.go b/internal/indexbuilder/task.go index 15647e1563..73a5b77508 100644 --- a/internal/indexbuilder/task.go +++ b/internal/indexbuilder/task.go @@ -2,6 +2,7 @@ package indexbuilder import ( "context" + "fmt" "log" "strconv" "time" @@ -171,10 +172,12 @@ func (it *IndexBuildTask) Execute() error { indexParams[key] = value } + fmt.Println("before NewCIndex ..........................") it.index, err = NewCIndex(typeParams, indexParams) if err != nil { return err } + fmt.Println("after NewCIndex ..........................") getKeyByPathNaive := func(path string) string { // splitElements := strings.Split(path, "/") @@ -223,6 +226,7 @@ func (it *IndexBuildTask) Execute() error { for _, value := range insertData.Data { // TODO: BinaryVectorFieldData + fmt.Println("before build index ..................................") floatVectorFieldData, fOk := value.(*storage.FloatVectorFieldData) if fOk { err = it.index.BuildFloatVecIndexWithoutIds(floatVectorFieldData.Data) @@ -238,12 +242,15 @@ func (it *IndexBuildTask) Execute() error { return err } } + fmt.Println("after build index ..................................") if !fOk && !bOk { return errors.New("we expect FloatVectorFieldData or BinaryVectorFieldData") } + fmt.Println("before serialize .............................................") indexBlobs, err := it.index.Serialize() + fmt.Println("after serialize .............................................") if err != nil { return err } diff --git a/internal/master/index_task.go b/internal/master/index_task.go index c5f320d0db..444b777c9f 100644 --- a/internal/master/index_task.go +++ b/internal/master/index_task.go @@ -24,6 +24,11 @@ func (task *createIndexTask) Ts() (Timestamp, error) { } func (task *createIndexTask) Execute() error { + // modify schema + if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil { + return err + } + // check if closed segment has the same index build history collMeta, err := task.mt.GetCollectionByName(task.req.CollectionName) if err != nil { return err @@ -39,20 +44,6 @@ func (task *createIndexTask) Execute() error { return fmt.Errorf("can not find field name %s", task.req.FieldName) } - // pre checks - isIndexable, err := task.mt.IsIndexable(collMeta.ID, fieldID) - if err != nil { - return err - } - if !isIndexable { - return fmt.Errorf("field %s is not vector", task.req.FieldName) - } - - // modify schema - if err := task.mt.UpdateFieldIndexParams(task.req.CollectionName, task.req.FieldName, task.req.ExtraParams); err != nil { - return err - } - // check if closed segment has the same index build history for _, segID := range collMeta.SegmentIDs { segMeta, err := task.mt.GetSegmentByID(segID) if err != nil { diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 38ad83b7e3..3993e6aae5 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -488,9 +488,7 @@ func TestProxy_CreateIndex(t *testing.T) { go func(group *sync.WaitGroup) { defer group.Done() createCollection(t, collName) - if i%2 == 0 { - createIndex(t, collName, fieldName) - } + createIndex(t, collName, fieldName) dropCollection(t, collName) // dropIndex(t, collectionName, fieldName, indexName) }(&wg) @@ -512,9 +510,7 @@ func TestProxy_DescribeIndex(t *testing.T) { go func(group *sync.WaitGroup) { defer group.Done() createCollection(t, collName) - if i%2 == 0 { - createIndex(t, collName, fieldName) - } + createIndex(t, collName, fieldName) req := &servicepb.DescribeIndexRequest{ CollectionName: collName, FieldName: fieldName, @@ -543,9 +539,7 @@ func TestProxy_DescribeIndexProgress(t *testing.T) { go func(group *sync.WaitGroup) { defer group.Done() createCollection(t, collName) - if i%2 == 0 { - createIndex(t, collName, fieldName) - } + createIndex(t, collName, fieldName) req := &servicepb.DescribeIndexProgressRequest{ CollectionName: collName, FieldName: fieldName, diff --git a/scripts/init_devcontainer.sh b/scripts/init_devcontainer.sh index afbca6fd34..80321670b9 100755 --- a/scripts/init_devcontainer.sh +++ b/scripts/init_devcontainer.sh @@ -34,6 +34,8 @@ awk 'c&&c--{sub(/^/,"#")} /# Build devcontainer/{c=5} 1' $ROOT_DIR/docker-compos awk 'c&&c--{sub(/^/,"#")} /# Command/{c=3} 1' $ROOT_DIR/docker-compose-vscode.yml.tmp > $ROOT_DIR/docker-compose-vscode.yml +rm $ROOT_DIR/docker-compose-vscode.yml.tmp + if [ "${machine}" == "Mac" ];then sed -i '' "s/# user: {{ CURRENT_ID }}/user: \"$uid:$gid\"/g" $ROOT_DIR/docker-compose-vscode.yml else