milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Upgrade confluent kafka go version to 1.9.1 (#18293) 

Signed-off-by: yun.zhang <yun.zhang@zilliz.com>
pull/18374/head
jaime 2022-07-22 13:28:28 +08:00 committed by GitHub
parent 9672eae62c
commit 3ec147acfe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 65 additions and 246 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Makefile
View File

@ -96,7 +96,6 @@ print-build-info:
milvus: build-cpp print-build-info
@echo "Building Milvus ..."
@echo "if build fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@source $(PWD)/scripts/setenv.sh && \
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
@ -104,7 +103,6 @@ milvus: build-cpp print-build-info
embd-milvus: build-cpp-embd print-build-info
@echo "Building **Embedded** Milvus ..."
@echo "if build fails on Mac M1 machines, rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@source $(PWD)/scripts/setenv.sh && \
mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \
GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \
@ -131,7 +129,6 @@ build-cpp-with-coverage:
# Run the tests.
unittest: test-cpp test-go
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
test-indexnode:
@echo "Running go unittests..."
@ -160,28 +157,23 @@ test-querycoord:
test-go: build-cpp-with-unittest
@echo "Running go unittests..."
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@(env bash $(PWD)/scripts/run_go_unittest.sh)
test-cpp: build-cpp-with-unittest
@echo "Running cpp unittests..."
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@(env bash $(PWD)/scripts/run_cpp_unittest.sh)
# Run code coverage.
codecov: codecov-go codecov-cpp
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
# Run codecov-go
codecov-go: build-cpp-with-coverage
@echo "Running go coverage..."
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@(env bash $(PWD)/scripts/run_go_codecov.sh)
# Run codecov-cpp
codecov-cpp: build-cpp-with-coverage
@echo "Running cpp coverage..."
@echo "if test fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`"
@(env bash $(PWD)/scripts/run_cpp_codecov.sh)
# Package docker image locally.

11
go.mod
View File

@ -7,13 +7,12 @@ require (
github.com/BurntSushi/toml v1.0.0
github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e // indirect
github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20210826220005-b48c857c3a0e
github.com/antonmedv/expr v1.8.9
github.com/apache/arrow/go/v8 v8.0.0-20220322092137-778b1772fd20
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7
github.com/apache/thrift v0.15.0
github.com/bits-and-blooms/bloom/v3 v3.0.1
github.com/confluentinc/confluent-kafka-go v1.8.2
github.com/confluentinc/confluent-kafka-go v1.9.1
github.com/containerd/cgroups v1.0.2
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@ -22,8 +21,6 @@ require (
github.com/go-basic/ipv4 v1.0.0
github.com/go-sql-driver/mysql v1.6.0 // indirect
github.com/gofrs/flock v0.8.1
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang/mock v1.5.0
github.com/golang/protobuf v1.5.2
github.com/google/btree v1.0.1
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
@ -58,8 +55,8 @@ require (
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519
golang.org/x/exp v0.0.0-20211216164055-b2b84827b756
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
google.golang.org/grpc v1.44.0
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f // indirect
google.golang.org/grpc v1.46.0
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f
gopkg.in/natefinch/lumberjack.v2 v2.0.0
stathat.com/c/consistent v1.0.0
)

283
go.sum
View File

File diff suppressed because it is too large Load Diff

9
internal/mq/msgstream/mqwrapper/kafka/kafka_consumer.go
View File

@ -228,13 +228,10 @@ func (kc *Consumer) GetLatestMsgID() (mqwrapper.MessageID, error) {
func (kc *Consumer) Close() {
kc.closeOnce.Do(func() {
start := time.Now()
// FIXME we should not use goroutine to close consumer, it will be fix after pr https://github.com/confluentinc/confluent-kafka-go/pull/757
go kc.c.Close()
kc.c.Close()
cost := time.Since(start).Milliseconds()
if cost > 500 {
log.Debug("kafka consumer is closed ", zap.Any("topic", kc.topic), zap.String("groupID", kc.groupID), zap.Int64("time cost(ms)", cost))
if cost > 200 {
log.Warn("close consumer costs too long time", zap.Any("topic", kc.topic), zap.String("groupID", kc.groupID), zap.Int64("time(ms)", cost))
}
})
}