milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Support Woodpecker as a WAL storage option for Milvus (#41095) 

#40916 Support Woodpecker as a WAL storage option for Milvus

Signed-off-by: tinswzy <zhenyuan.wei@zilliz.com>
pull/41290/head
tinswzy 2025-04-20 22:22:42 +08:00 committed by GitHub
parent c893344289
commit 6fa68c1f16
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
25 changed files with 1233 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cmd/tools/config/generate.go
View File

@ -238,6 +238,11 @@ func WriteYaml(w io.Writer) {
# If you don't set mq.type field as default, there is a note about enabling priority if we config multiple mq in this file.
# 1. standalone(local) mode: rocksmq(default) > natsmq > Pulsar > Kafka
# 2. cluster mode: Pulsar(default) > Kafka (rocksmq and natsmq is unsupported in cluster mode)`,
},
{
name: "woodpecker",
header: `
# Related configuration of woodpecker, used to manage Milvus logs of recent mutation operations, output streaming log, and provide embedded log sequential read and write.`,
},
{
name: "pulsar",

32
configs/milvus.yaml
View File

@ -165,7 +165,7 @@ minio:
# 2. cluster mode: Pulsar(default) > Kafka (rocksmq and natsmq is unsupported in cluster mode)
mq:
# Default value: "default"
# Valid values: [default, pulsar, kafka, rocksmq, natsmq]
# Valid values: [default, pulsar, kafka, rocksmq, natsmq, woodpecker]
type: default
enablePursuitMode: true # Default value: "true"
pursuitLag: 10 # time tick lag threshold to enter pursuit mode, in seconds
@ -177,6 +177,36 @@ mq:
targetBufSize: 16 # the lenth of channel buffer for targe
maxTolerantLag: 3 # Default value: "3", the timeout(in seconds) that target sends msgPack
# Related configuration of woodpecker, used to manage Milvus logs of recent mutation operations, output streaming log, and provide embedded log sequential read and write.
woodpecker:
meta:
type: etcd # The Type of the metadata provider. currently only support etcd.
prefix: woodpecker # The Prefix of the metadata provider. default is woodpecker.
client:
segmentAppend:
queueSize: 10000 # The size of the queue for pending messages to be sent of each log.
maxRetries: 3 # Maximum number of retries for segment append operations.
segmentRollingPolicy:
maxSize: 2000000000 # Maximum entries count of a segment, default is 2GB
maxInterval: 600 # Maximum interval between two segments in seconds, default is 10 minutes.
auditor:
maxInterval: 10 # Maximum interval between two auditing operations in seconds, default is 10 seconds.
logstore:
logFileSyncPolicy:
maxInterval: 1000 # Maximum interval between two sync operations in milliseconds.
maxBytes: 64000000 # Maximum size of write buffer in bytes.
maxEntries: 100000 # Maximum entries number of write buffer.
maxFlushRetries: 5 # Maximum size of write buffer in bytes.
retryInterval: 1000 # Maximum interval between two retries in milliseconds.
maxFlushSize: 8000000 # Maximum size of a fragment in bytes to flush, default is 8M.
maxFlushThreads: 4 # Maximum number of threads to flush data
fragmentManager:
maxBytes: 1000000000 # Maximum size of fragment cached data in bytes.
maxInterval: 1000 # Maximum interval between two fragment evicts in milliseconds.
storage:
type: minio # The Type of the storage provider. Valid values: [default, minio, local, service], default is minio.
rootPath: /var/lib/milvus/woodpecker # The root path of the storage provider.
# Related configuration of pulsar, used to manage Milvus logs of recent mutation operations, output streaming log, and provide log publish-subscribe services.
pulsar:
# IP address of Pulsar service.

11
go.mod
View File

@ -18,7 +18,7 @@ require (
github.com/gin-gonic/gin v1.9.1
github.com/go-playground/validator/v10 v10.14.0
github.com/gofrs/flock v0.8.1
github.com/golang/protobuf v1.5.4
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/btree v1.1.2
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.17.9
@ -140,6 +140,7 @@ require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/ebitengine/purego v0.8.1 // indirect
github.com/edsrzf/mmap-go v1.2.0 // indirect
github.com/expr-lang/expr v1.15.7 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
@ -182,6 +183,7 @@ require (
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/labstack/gommon v0.3.0 // indirect
github.com/leodido/go-urn v1.2.4 // indirect
github.com/linkedin/goavro/v2 v2.11.1 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@ -214,6 +216,7 @@ require (
github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 // indirect
github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/procfs v0.9.0 // indirect
@ -223,7 +226,8 @@ require (
github.com/rs/xid v1.5.0 // indirect
github.com/sasha-s/go-deadlock v0.3.1 // indirect
github.com/shirou/gopsutil v3.20.11+incompatible // indirect
github.com/shirou/gopsutil/v3 v3.22.9 // indirect
github.com/shirou/gopsutil/v3 v3.23.7 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
@ -245,9 +249,12 @@ require (
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect

34
go.sum
View File

@ -275,6 +275,8 @@ github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA
github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
github.com/ebitengine/purego v0.8.1 h1:sdRKd6plj7KYW33EH5As6YKfe8m9zbN9JMrOjNVF/BE=
github.com/ebitengine/purego v0.8.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -472,6 +474,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -674,6 +678,7 @@ github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b/go.mod h1:V0HF
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
@ -842,6 +847,8 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -924,10 +931,14 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil v3.20.11+incompatible h1:LJr4ZQK4mPpIV5gOa4jCOKOGb4ty4DZO54I4FGqIpto=
github.com/shirou/gopsutil v3.20.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA=
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
github.com/shirou/gopsutil/v4 v4.24.10 h1:7VOzPtfw/5YDU+jLEoBwXwxJbQetULywoSV4RYY7HkM=
github.com/shirou/gopsutil/v4 v4.24.10/go.mod h1:s4D/wg+ag4rG0WO7AiTj2BeYCRhym0vM7DHbZRxnIT8=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
@ -955,8 +966,8 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
@ -988,6 +999,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
@ -1009,10 +1021,10 @@ github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2X
github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -1033,12 +1045,14 @@ github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLY
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasthttp v1.14.1-0.20200605121233-ac51d598dc54/go.mod h1:hWrFNtR2Jc1XrK0fDq2Y+MkA7F/v3lYKRDXd2CmSikc=
github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ=
github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@ -1060,13 +1074,15 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0=
github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 h1:Te+TxCQVisH/ntsulwDlO77s3PuJhj//ok9xnaGupZo=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
@ -1390,15 +1406,15 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

2
internal/streamingnode/server/server.go
View File

@ -64,7 +64,7 @@ func (s *Server) initBasicComponent() {
var err error
s.walManager, err = walmanager.OpenManager()
if err != nil {
panic("open wal manager failed")
panic(fmt.Sprintf("open wal manager failed, %+v", err))
}
// Register the wal manager to the local registry.
registry.RegisterLocalWALManager(s.walManager)

32
internal/util/dependency/factory.go
View File

@ -16,18 +16,20 @@ import (
)
const (
mqTypeDefault = "default"
mqTypeNatsmq = "natsmq"
mqTypeRocksmq = "rocksmq"
mqTypeKafka = "kafka"
mqTypePulsar = "pulsar"
mqTypeDefault = "default"
mqTypeNatsmq = "natsmq"
mqTypeRocksmq = "rocksmq"
mqTypeKafka = "kafka"
mqTypePulsar = "pulsar"
mqTypeWoodpecker = "woodpecker"
)
type mqEnable struct {
Rocksmq bool
Natsmq bool
Pulsar bool
Kafka bool
Rocksmq bool
Natsmq bool
Pulsar bool
Kafka bool
Woodpecker bool
}
// DefaultFactory is a factory that produces instances of storage.ChunkManager and message queue.
@ -82,7 +84,7 @@ func (f *DefaultFactory) Init(params *paramtable.ComponentParam) {
}
func (f *DefaultFactory) initMQ(standalone bool, params *paramtable.ComponentParam) error {
mqType := mustSelectMQType(standalone, params.MQCfg.Type.GetValue(), mqEnable{params.RocksmqEnable(), params.NatsmqEnable(), params.PulsarEnable(), params.KafkaEnable()})
mqType := mustSelectMQType(standalone, params.MQCfg.Type.GetValue(), mqEnable{params.RocksmqEnable(), params.NatsmqEnable(), params.PulsarEnable(), params.KafkaEnable(), params.WoodpeckerEnable()})
metrics.RegisterMQType(mqType)
log.Info("try to init mq", zap.Bool("standalone", standalone), zap.String("mqType", mqType))
@ -95,6 +97,8 @@ func (f *DefaultFactory) initMQ(standalone bool, params *paramtable.ComponentPar
f.msgStreamFactory = msgstream.NewPmsFactory(&params.ServiceParam)
case mqTypeKafka:
f.msgStreamFactory = msgstream.NewKmsFactory(&params.ServiceParam)
case mqTypeWoodpecker:
f.msgStreamFactory = msgstream.NewWpmsFactory(&params.ServiceParam)
}
if f.msgStreamFactory == nil {
return errors.New("failed to create MQ: check the milvus log for initialization failures")
@ -122,13 +126,16 @@ func mustSelectMQType(standalone bool, mqType string, enable mqEnable) string {
if enable.Kafka {
return mqTypeKafka
}
if enable.Woodpecker {
return mqTypeWoodpecker
}
panic(errors.Errorf("no available mq config found, %s, enable: %+v", mqType, enable))
}
// Validate mq type.
func validateMQType(standalone bool, mqType string) error {
if mqType != mqTypeNatsmq && mqType != mqTypeRocksmq && mqType != mqTypeKafka && mqType != mqTypePulsar {
if mqType != mqTypeNatsmq && mqType != mqTypeRocksmq && mqType != mqTypeKafka && mqType != mqTypePulsar && mqType != mqTypeWoodpecker {
return errors.Newf("mq type %s is invalid", mqType)
}
if !standalone && (mqType == mqTypeRocksmq || mqType == mqTypeNatsmq) {
@ -172,6 +179,9 @@ func HealthCheck(mqType string) *common.MQClusterStatus {
msgstream.PulsarHealthCheck(clusterStatus)
case mqTypeKafka:
msgstream.KafkaHealthCheck(clusterStatus)
case mqTypeWoodpecker:
// TODO: implement health checker for woodpecker
clusterStatus.Health = true
}
return clusterStatus
}

43
internal/util/dependency/factory_test.go
View File

@ -13,27 +13,33 @@ func TestValidateMQType(t *testing.T) {
assert.Error(t, validateMQType(false, mqTypeDefault))
assert.Error(t, validateMQType(false, mqTypeNatsmq))
assert.Error(t, validateMQType(false, mqTypeRocksmq))
assert.NoError(t, validateMQType(true, mqTypeWoodpecker))
assert.NoError(t, validateMQType(false, mqTypeWoodpecker))
}
func TestSelectMQType(t *testing.T) {
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{true, true, true, true}), mqTypeRocksmq)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, false, true}), mqTypeKafka)
assert.Panics(t, func() { mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, false, false}) })
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, false, true}), mqTypeKafka)
assert.Panics(t, func() { mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, false, false}) })
assert.Equal(t, mustSelectMQType(true, mqTypeRocksmq, mqEnable{true, true, true, true}), mqTypeRocksmq)
assert.Equal(t, mustSelectMQType(true, mqTypeNatsmq, mqEnable{true, true, true, true}), mqTypeNatsmq)
assert.Equal(t, mustSelectMQType(true, mqTypePulsar, mqEnable{true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeKafka, mqEnable{true, true, true, true}), mqTypeKafka)
assert.Panics(t, func() { mustSelectMQType(false, mqTypeRocksmq, mqEnable{true, true, true, true}) })
assert.Panics(t, func() { mustSelectMQType(false, mqTypeNatsmq, mqEnable{true, true, true, true}) })
assert.Equal(t, mustSelectMQType(false, mqTypePulsar, mqEnable{true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeKafka, mqEnable{true, true, true, true}), mqTypeKafka)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{true, true, true, true, true}), mqTypeRocksmq)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, false, true, true}), mqTypeKafka)
assert.Equal(t, mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, false, false, true}), mqTypeWoodpecker)
assert.Panics(t, func() { mustSelectMQType(true, mqTypeDefault, mqEnable{false, false, false, false, false}) })
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{true, true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, false, true, true}), mqTypeKafka)
assert.Equal(t, mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, false, false, true}), mqTypeWoodpecker)
assert.Panics(t, func() { mustSelectMQType(false, mqTypeDefault, mqEnable{false, false, false, false, false}) })
assert.Equal(t, mustSelectMQType(true, mqTypeRocksmq, mqEnable{true, true, true, true, true}), mqTypeRocksmq)
assert.Equal(t, mustSelectMQType(true, mqTypeNatsmq, mqEnable{true, true, true, true, true}), mqTypeNatsmq)
assert.Equal(t, mustSelectMQType(true, mqTypePulsar, mqEnable{true, true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(true, mqTypeKafka, mqEnable{true, true, true, true, true}), mqTypeKafka)
assert.Equal(t, mustSelectMQType(true, mqTypeWoodpecker, mqEnable{true, true, true, true, true}), mqTypeWoodpecker)
assert.Panics(t, func() { mustSelectMQType(false, mqTypeRocksmq, mqEnable{true, true, true, true, true}) })
assert.Panics(t, func() { mustSelectMQType(false, mqTypeNatsmq, mqEnable{true, true, true, true, true}) })
assert.Equal(t, mustSelectMQType(false, mqTypePulsar, mqEnable{true, true, true, true, true}), mqTypePulsar)
assert.Equal(t, mustSelectMQType(false, mqTypeKafka, mqEnable{true, true, true, true, true}), mqTypeKafka)
assert.Equal(t, mustSelectMQType(false, mqTypeWoodpecker, mqEnable{true, true, true, true, true}), mqTypeWoodpecker)
}
func TestHealthCheck(t *testing.T) {
@ -51,6 +57,7 @@ func TestHealthCheck(t *testing.T) {
{mqTypeRocksmq, true},
{mqTypePulsar, false},
{mqTypeKafka, false},
{mqTypeWoodpecker, true},
{"invalidType", false},
}

20
internal/util/streamingutil/util/wal_selector.go
View File

@ -8,16 +8,18 @@ import (
)
const (
walTypeDefault = "default"
walTypeRocksmq = "rocksmq"
walTypeKafka = "kafka"
walTypePulsar = "pulsar"
walTypeDefault = "default"
walTypeRocksmq = "rocksmq"
walTypeKafka = "kafka"
walTypePulsar = "pulsar"
walTypeWoodpecker = "woodpecker"
)
type walEnable struct {
Rocksmq bool
Pulsar bool
Kafka bool
Rocksmq bool
Pulsar bool
Kafka bool
Woodpecker bool
}
// MustSelectWALName select wal name.
@ -28,6 +30,7 @@ func MustSelectWALName() string {
params.RocksmqEnable(),
params.PulsarEnable(),
params.KafkaEnable(),
params.WoodpeckerEnable(),
})
}
@ -50,6 +53,9 @@ func mustSelectWALName(standalone bool, mqType string, enable walEnable) string
if enable.Kafka {
return walTypeKafka
}
if enable.Woodpecker {
return walTypeWoodpecker
}
panic(errors.Errorf("no available wal config found, %s, enable: %+v", mqType, enable))
}

35
internal/util/streamingutil/util/wal_selector_test.go
View File

@ -11,20 +11,23 @@ func TestValidateWALType(t *testing.T) {
}
func TestSelectWALType(t *testing.T) {
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{true, true, true}), walTypeRocksmq)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, false, true}), walTypeKafka)
assert.Panics(t, func() { mustSelectWALName(true, walTypeDefault, walEnable{false, false, false}) })
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, false, true}), walTypeKafka)
assert.Panics(t, func() { mustSelectWALName(false, walTypeDefault, walEnable{false, false, false}) })
assert.Equal(t, mustSelectWALName(true, walTypeRocksmq, walEnable{true, true, true}), walTypeRocksmq)
assert.Equal(t, mustSelectWALName(true, walTypePulsar, walEnable{true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(true, walTypeKafka, walEnable{true, true, true}), walTypeKafka)
assert.Panics(t, func() { mustSelectWALName(false, walTypeRocksmq, walEnable{true, true, true}) })
assert.Equal(t, mustSelectWALName(false, walTypePulsar, walEnable{true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeKafka, walEnable{true, true, true}), walTypeKafka)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{true, true, true, true}), walTypeRocksmq)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, false, true, true}), walTypeKafka)
assert.Equal(t, mustSelectWALName(true, walTypeDefault, walEnable{false, false, false, true}), walTypeWoodpecker)
assert.Panics(t, func() { mustSelectWALName(true, walTypeDefault, walEnable{false, false, false, false}) })
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{true, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, false, true, true}), walTypeKafka)
assert.Equal(t, mustSelectWALName(false, walTypeDefault, walEnable{false, false, false, true}), walTypeWoodpecker)
assert.Panics(t, func() { mustSelectWALName(false, walTypeDefault, walEnable{false, false, false, false}) })
assert.Equal(t, mustSelectWALName(true, walTypeRocksmq, walEnable{true, true, true, true}), walTypeRocksmq)
assert.Equal(t, mustSelectWALName(true, walTypePulsar, walEnable{true, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(true, walTypeKafka, walEnable{true, true, true, true}), walTypeKafka)
assert.Equal(t, mustSelectWALName(true, walTypeWoodpecker, walEnable{true, true, true, true}), walTypeWoodpecker)
assert.Panics(t, func() { mustSelectWALName(false, walTypeRocksmq, walEnable{true, true, true, true}) })
assert.Equal(t, mustSelectWALName(false, walTypePulsar, walEnable{true, true, true, true}), walTypePulsar)
assert.Equal(t, mustSelectWALName(false, walTypeKafka, walEnable{true, true, true, true}), walTypeKafka)
assert.Equal(t, mustSelectWALName(false, walTypeWoodpecker, walEnable{true, true, true, true}), walTypeWoodpecker)
}

31
pkg/go.mod
View File

@ -1,6 +1,6 @@
module github.com/milvus-io/milvus/pkg/v2
go 1.21
go 1.22
require (
cloud.google.com/go/storage v1.43.0
@ -30,8 +30,8 @@ require (
github.com/remeh/sizedwaitgroup v1.0.0
github.com/samber/lo v1.27.0
github.com/sasha-s/go-deadlock v0.3.1
github.com/shirou/gopsutil/v3 v3.22.9
github.com/sirupsen/logrus v1.9.0
github.com/shirou/gopsutil/v3 v3.23.7
github.com/sirupsen/logrus v1.9.3
github.com/spaolacci/murmur3 v1.1.0
github.com/spf13/cast v1.3.1
github.com/streamnative/pulsarctl v0.5.0
@ -42,6 +42,7 @@ require (
github.com/tikv/client-go/v2 v2.0.4
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65
go.etcd.io/etcd/api/v3 v3.5.5
go.etcd.io/etcd/client/v3 v3.5.5
go.etcd.io/etcd/server/v3 v3.5.5
@ -53,11 +54,11 @@ require (
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0
go.opentelemetry.io/otel/sdk v1.28.0
go.opentelemetry.io/otel/trace v1.28.0
go.uber.org/atomic v1.10.0
go.uber.org/atomic v1.11.0
go.uber.org/automaxprocs v1.5.3
go.uber.org/zap v1.27.0
golang.org/x/crypto v0.31.0
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842
golang.org/x/net v0.33.0
golang.org/x/oauth2 v0.21.0
golang.org/x/sync v0.10.0
@ -96,11 +97,12 @@ require (
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/danieljoos/wincred v1.1.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
github.com/edsrzf/mmap-go v1.2.0 // indirect
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
@ -138,8 +140,10 @@ require (
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/labstack/gommon v0.3.0 // indirect
github.com/linkedin/goavro/v2 v2.11.1 // indirect
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mattn/go-runewidth v0.0.8 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
@ -162,7 +166,8 @@ require (
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
@ -170,7 +175,9 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/rs/xid v1.5.0 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/soheilhy/cmux v0.1.5 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
@ -178,12 +185,14 @@ require (
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 // indirect
github.com/tklauser/go-sysconf v0.3.10 // indirect
github.com/tklauser/numcpus v0.4.0 // indirect
github.com/tklauser/go-sysconf v0.3.11 // indirect
github.com/tklauser/numcpus v0.6.0 // indirect
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
@ -203,7 +212,7 @@ require (
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.56.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

76
pkg/go.sum
View File

@ -175,6 +175,7 @@ github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso=
@ -182,8 +183,9 @@ github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs
github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@ -198,6 +200,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY=
github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU=
github.com/edsrzf/mmap-go v1.2.0 h1:hXLYlkbaPzt1SaQk+anYwKSRNhufIDCchSPkUD6dD84=
github.com/edsrzf/mmap-go v1.2.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q=
github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@ -356,6 +360,8 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/gops v0.3.28 h1:2Xr57tqKAmQYRAfG12E+yLcoa2Y42UJo2lOrUFL9ark=
github.com/google/gops v0.3.28/go.mod h1:6f6+Nl8LcHrzJwi8+p0ii+vmBFSlB4f8cOOkTJ7sk4c=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -388,7 +394,6 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBYGmXdxA=
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 h1:l5lAOZEym3oK3SQ2HBHWsJUfbNBiTXJDeW2QDxw9AQ0=
github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
@ -441,8 +446,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/invopop/jsonschema v0.4.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0=
github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI=
github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0=
@ -471,7 +476,6 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@ -513,6 +517,7 @@ github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b/go.mod h1:V0HF
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
@ -528,14 +533,16 @@ github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPK
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.11 h1:nQ+aFkoE2TMGc0b68U2OKSexC+eq46+XwZzWXHRmPYs=
github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0=
@ -641,8 +648,11 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo=
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
@ -697,6 +707,7 @@ github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
github.com/samber/lo v1.27.0 h1:GOyDWxsblvqYobqsmUuMddPa2/mMzkKyojlXol4+LaQ=
@ -708,19 +719,21 @@ github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtm
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4c5eA=
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shirou/gopsutil/v3 v3.23.7 h1:C+fHO8hfIppoJ1WdsVm1RoI0RwXoNdfTK7yWXV0wVj4=
github.com/shirou/gopsutil/v3 v3.23.7/go.mod h1:c4gnmoRC0hQuaLqvxnx1//VXQ0Ms/X9UnJF8pddY5z4=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/assertions v1.1.0 h1:MkTeG1DMwsrdH7QtLXy5W+fUxWq+vmb6cLmyJ7aRtF0=
github.com/smartystreets/assertions v1.1.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
@ -734,8 +747,8 @@ github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@ -761,6 +774,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@ -780,10 +794,10 @@ github.com/tikv/client-go/v2 v2.0.4 h1:cPtMXTExqjzk8L40qhrgB/mXiBXKP5LRU0vwjtI2X
github.com/tikv/client-go/v2 v2.0.4/go.mod h1:v52O5zDtv2BBus4lm5yrSQhxGW4Z4RaXWfg0U1Kuyqo=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07 h1:ckPpxKcl75mO2N6a4cJXiZH43hvcHPpqc9dh1TmH1nc=
github.com/tikv/pd/client v0.0.0-20221031025758-80f0d8ca4d07/go.mod h1:CipBxPfxPUME+BImx9MUYXCnAVLS3VJUr3mnSJwh40A=
github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw=
github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o=
github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@ -796,9 +810,11 @@ github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVM
github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4=
github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
@ -820,8 +836,10 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65 h1:Te+TxCQVisH/ntsulwDlO77s3PuJhj//ok9xnaGupZo=
github.com/zilliztech/woodpecker v0.0.0-20250418010644-1a9ae136fa65/go.mod h1:MLt2hsMXd5bVOykwZyWXYHsy9kN4C2gQEaCrID5rM1w=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
@ -887,8 +905,8 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8=
go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0=
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
@ -931,8 +949,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2 h1:Jvc7gsqn21cJHCmAWx0LiimpP18LZmUxkT5Mp7EZ1mI=
golang.org/x/exp v0.0.0-20230224173230-c95f2b4c22f2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@ -1105,13 +1123,14 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -1346,8 +1365,9 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.56.0 h1:DPMeDvGTM54DXbPkVIZsp19fp/I2K7zwA/itHYHKo8Y=
gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=

77
pkg/mq/msgstream/mq_factory.go
View File

@ -250,3 +250,80 @@ func NewRocksmqFactory(path string, cfg *paramtable.ServiceParam) Factory {
MQBufSize: cfg.MQCfg.MQBufSize.GetAsInt64(),
}
}
var _ Factory = &WpmsFactory{}
// TODO Should use streamingNode uniformly as a message stream service
type WpmsFactory struct{}
func (w WpmsFactory) NewMsgStream(ctx context.Context) (MsgStream, error) {
return &WpMsgStream{}, nil
}
func (w WpmsFactory) NewTtMsgStream(ctx context.Context) (MsgStream, error) {
return nil, nil
}
func (w WpmsFactory) NewMsgStreamDisposer(ctx context.Context) func([]string, string) error {
return nil
}
var _ MsgStream = &WpMsgStream{}
type WpMsgStream struct{}
func (w WpMsgStream) Close() {
// NO-OP
}
func (w WpMsgStream) AsProducer(ctx context.Context, channels []string) {
// NO-OP
}
func (w WpMsgStream) Produce(ctx context.Context, pack *MsgPack) error {
// NO-OP
return nil
}
func (w WpMsgStream) SetRepackFunc(repackFunc RepackFunc) {}
func (w WpMsgStream) GetProduceChannels() []string {
return nil
}
func (w WpMsgStream) Broadcast(ctx context.Context, pack *MsgPack) (map[string][]MessageID, error) {
return nil, nil
}
func (w WpMsgStream) AsConsumer(ctx context.Context, channels []string, subName string, position common.SubscriptionInitialPosition) error {
return nil
}
func (w WpMsgStream) Chan() <-chan *ConsumeMsgPack {
return nil
}
func (w WpMsgStream) GetUnmarshalDispatcher() UnmarshalDispatcher {
return nil
}
func (w WpMsgStream) Seek(ctx context.Context, msgPositions []*MsgPosition, includeCurrentMsg bool) error {
return nil
}
func (w WpMsgStream) GetLatestMsgID(channel string) (MessageID, error) {
return nil, nil
}
func (w WpMsgStream) CheckTopicValid(channel string) error {
return nil
}
func (w WpMsgStream) ForceEnableProduce(can bool) {
}
// NewWpmsFactory creates a new message stream factory based on woodpecker
func NewWpmsFactory(cfg *paramtable.ServiceParam) Factory {
// TODO should not be used in mq wrapper
return &WpmsFactory{}
}

60
pkg/mq/msgstream/mq_factory_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/v2/mq/common"
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
)
@ -167,3 +168,62 @@ func TestRmsFactory(t *testing.T) {
_, err = rmsFactory.NewTtMsgStream(ctx)
assert.NoError(t, err)
}
func TestWpmsFactory(t *testing.T) {
wpmsFactory := NewWpmsFactory(&Params.ServiceParam)
ctx := context.Background()
// Test NewMsgStream
stream, err := wpmsFactory.NewMsgStream(ctx)
assert.NoError(t, err)
assert.NotNil(t, stream)
// Test NewTtMsgStream
ttStream, err := wpmsFactory.NewTtMsgStream(ctx)
assert.NoError(t, err)
assert.Nil(t, ttStream)
// Test NewMsgStreamDisposer
disposer := wpmsFactory.NewMsgStreamDisposer(ctx)
assert.Nil(t, disposer)
}
func TestWpMsgStream(t *testing.T) {
wpStream := &WpMsgStream{}
// Test methods return expected values
assert.Nil(t, wpStream.GetProduceChannels())
assert.Nil(t, wpStream.Chan())
assert.Nil(t, wpStream.GetUnmarshalDispatcher())
// Test methods execute without panic
ctx := context.Background()
// Test all no-op methods
wpStream.Close()
wpStream.AsProducer(ctx, []string{"test-channel"})
wpStream.SetRepackFunc(nil)
wpStream.ForceEnableProduce(true)
// Test methods returning nil/empty values
msgID, err := wpStream.GetLatestMsgID("test-channel")
assert.Nil(t, msgID)
assert.Nil(t, err)
err = wpStream.CheckTopicValid("test-channel")
assert.Nil(t, err)
err = wpStream.Produce(ctx, &MsgPack{})
assert.Nil(t, err)
broadcastResult, err := wpStream.Broadcast(ctx, &MsgPack{})
assert.Nil(t, broadcastResult)
assert.Nil(t, err)
err = wpStream.AsConsumer(ctx, []string{"test-channel"}, "test-sub", common.SubscriptionPositionEarliest)
assert.Nil(t, err)
err = wpStream.Seek(ctx, []*MsgPosition{}, false)
assert.Nil(t, err)
}

61
pkg/mq/msgstream/mqwrapper/wp/wp_id.go Normal file
View File

@ -0,0 +1,61 @@
package wp
import (
"github.com/zilliztech/woodpecker/woodpecker/log"
"github.com/milvus-io/milvus/pkg/v2/mq/common"
)
func NewWoodpeckerID(id *log.LogMessageId) *woodpeckerID {
return &woodpeckerID{
messageID: id,
}
}
type woodpeckerID struct {
messageID *log.LogMessageId
}
func (w *woodpeckerID) WoodpeckerID() *log.LogMessageId {
return w.messageID
}
func (w *woodpeckerID) Serialize() []byte {
return w.messageID.Serialize()
}
func (w *woodpeckerID) AtEarliestPosition() bool {
if w.messageID.SegmentId <= 0 && w.messageID.EntryId <= 0 {
return true
}
return false
}
func (w *woodpeckerID) LessOrEqualThan(msgID []byte) (bool, error) {
id2, err := log.DeserializeLogMessageId(msgID)
if err != nil {
return false, err
}
if w.messageID.SegmentId < id2.SegmentId {
return true, nil
}
if w.messageID.SegmentId > id2.SegmentId {
return false, nil
}
return w.messageID.EntryId <= id2.EntryId, nil
}
func (w *woodpeckerID) Equal(msgID []byte) (bool, error) {
id2, err := log.DeserializeLogMessageId(msgID)
if err != nil {
return false, err
}
return w.messageID.SegmentId == id2.SegmentId && w.messageID.EntryId == id2.EntryId, nil
}
// Check if pulsarID implements and MessageID interface
var _ common.MessageID = &woodpeckerID{}
func DeserializeWoodpeckerMsgID(messageID []byte) (*log.LogMessageId, error) {
return log.DeserializeLogMessageId(messageID)
}

19
pkg/streaming/util/message/adaptor/message_id.go
View File

@ -5,15 +5,18 @@ import (
"github.com/apache/pulsar-client-go/pulsar"
rawKafka "github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/zilliztech/woodpecker/woodpecker/log"
"github.com/milvus-io/milvus/pkg/v2/mq/common"
"github.com/milvus-io/milvus/pkg/v2/mq/mqimpl/rocksmq/server"
mqkafka "github.com/milvus-io/milvus/pkg/v2/mq/msgstream/mqwrapper/kafka"
mqpulsar "github.com/milvus-io/milvus/pkg/v2/mq/msgstream/mqwrapper/pulsar"
mqwoodpecker "github.com/milvus-io/milvus/pkg/v2/mq/msgstream/mqwrapper/wp"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
msgkafka "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/kafka"
msgpulsar "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/pulsar"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/rmq"
msgwoodpecker "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/wp"
)
// MustGetMQWrapperIDFromMessage converts message.MessageID to common.MessageID
@ -25,6 +28,8 @@ func MustGetMQWrapperIDFromMessage(messageID message.MessageID) common.MessageID
return &server.RmqID{MessageID: id.RmqID()}
} else if id, ok := messageID.(interface{ KafkaID() rawKafka.Offset }); ok {
return mqkafka.NewKafkaID(int64(id.KafkaID()))
} else if id, ok := messageID.(interface{ WoodpeckerID() *log.LogMessageId }); ok {
return mqwoodpecker.NewWoodpeckerID(id.WoodpeckerID())
}
panic("unsupported now")
}
@ -38,6 +43,8 @@ func MustGetMessageIDFromMQWrapperID(commonMessageID common.MessageID) message.M
return rmq.NewRmqID(id.MessageID)
} else if id, ok := commonMessageID.(*mqkafka.KafkaID); ok {
return msgkafka.NewKafkaID(rawKafka.Offset(id.MessageID))
} else if id, ok := commonMessageID.(interface{ WoodpeckerID() *log.LogMessageId }); ok {
return msgwoodpecker.NewWpID(id.WoodpeckerID())
}
return nil
}
@ -58,6 +65,12 @@ func DeserializeToMQWrapperID(msgID []byte, walName string) (common.MessageID, e
case "kafka":
kID := mqkafka.DeserializeKafkaID(msgID)
return mqkafka.NewKafkaID(kID), nil
case "woodpecker":
wID, err := mqwoodpecker.DeserializeWoodpeckerMsgID(msgID)
if err != nil {
return nil, err
}
return mqwoodpecker.NewWoodpeckerID(wID), nil
default:
return nil, fmt.Errorf("unsupported mq type %s", walName)
}
@ -78,6 +91,12 @@ func MustGetMessageIDFromMQWrapperIDBytes(walName string, msgIDBytes []byte) mes
case "kafka":
id := mqkafka.DeserializeKafkaID(msgIDBytes)
commonMsgID = mqkafka.NewKafkaID(id)
case "woodpecker":
msgID, err := mqwoodpecker.DeserializeWoodpeckerMsgID(msgIDBytes)
if err != nil {
panic(err)
}
commonMsgID = mqwoodpecker.NewWoodpeckerID(msgID)
default:
panic("unsupported now")
}

6
pkg/streaming/util/message/adaptor/message_id_test.go
View File

@ -5,10 +5,12 @@ import (
"github.com/apache/pulsar-client-go/pulsar"
"github.com/stretchr/testify/assert"
wp "github.com/zilliztech/woodpecker/woodpecker/log"
msgkafka "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/kafka"
msgpulsar "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/pulsar"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/rmq"
msgwoodpecker "github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/impls/wp"
)
func TestIDConvension(t *testing.T) {
@ -21,4 +23,8 @@ func TestIDConvension(t *testing.T) {
kafkaID := MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(msgkafka.NewKafkaID(1)))
assert.True(t, kafkaID.EQ(msgkafka.NewKafkaID(1)))
logMsgId := wp.EarliestLogMessageID()
wpID := MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(msgwoodpecker.NewWpID(&logMsgId)))
assert.True(t, wpID.EQ(msgwoodpecker.NewWpID(&logMsgId)))
}

173
pkg/streaming/walimpls/impls/wp/builder.go Normal file
View File

@ -0,0 +1,173 @@
package wp
import (
"context"
"fmt"
"github.com/minio/minio-go/v7"
"github.com/zilliztech/woodpecker/common/config"
wpMinioHandler "github.com/zilliztech/woodpecker/common/minio"
"github.com/zilliztech/woodpecker/woodpecker"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/v2/log"
"github.com/milvus-io/milvus/pkg/v2/objectstorage"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/registry"
"github.com/milvus-io/milvus/pkg/v2/util/etcd"
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
)
const (
walName = "woodpecker"
)
func init() {
// register the builder to the wal registry.
registry.RegisterBuilder(&builderImpl{})
// register the unmarshaler to the message registry.
message.RegisterMessageIDUnmsarshaler(walName, UnmarshalMessageID)
}
// builderImpl is the builder for woodpecker opener.
type builderImpl struct{}
// Name of the wal builder, should be a lowercase string.
func (b *builderImpl) Name() string {
return walName
}
// Build build a wal instance.
func (b *builderImpl) Build() (walimpls.OpenerImpls, error) {
cfg, err := b.getWpConfig()
if err != nil {
return nil, err
}
var minioHandler wpMinioHandler.MinioHandler
if cfg.Woodpecker.Storage.IsStorageMinio() {
minioCli, err := b.getMinioClient(context.TODO())
if err != nil {
return nil, err
}
minioHandler, err = wpMinioHandler.NewMinioHandlerWithClient(context.Background(), minioCli)
if err != nil {
return nil, err
}
log.Ctx(context.Background()).Info("create minio handler finish while building wp opener")
}
etcdCli, err := b.getEtcdClient(context.TODO())
if err != nil {
return nil, err
}
log.Ctx(context.Background()).Info("create etcd client finish while building wp opener")
wpClient, err := woodpecker.NewEmbedClient(context.Background(), cfg, etcdCli, minioHandler, true)
if err != nil {
return nil, err
}
log.Ctx(context.Background()).Info("build wp opener finish", zap.String("wpClientInstance", fmt.Sprintf("%p", wpClient)))
return &openerImpl{
c: wpClient,
}, nil
}
func (b *builderImpl) getWpConfig() (*config.Configuration, error) {
wpConfig, err := config.NewConfiguration()
if err != nil {
return nil, err
}
err = b.setCustomWpConfig(wpConfig, &paramtable.Get().WoodpeckerCfg)
if err != nil {
return nil, err
}
return wpConfig, nil
}
func (b *builderImpl) setCustomWpConfig(wpConfig *config.Configuration, cfg *paramtable.WoodpeckerConfig) error {
// set the rootPath as the prefix for wp object storage
wpConfig.Woodpecker.Meta.Prefix = fmt.Sprintf("%s/wp", paramtable.Get().EtcdCfg.RootPath.GetValue())
// logClient
wpConfig.Woodpecker.Client.Auditor.MaxInterval = cfg.AuditorMaxInterval.GetAsInt()
wpConfig.Woodpecker.Client.SegmentAppend.MaxRetries = cfg.AppendMaxRetries.GetAsInt()
wpConfig.Woodpecker.Client.SegmentAppend.QueueSize = cfg.AppendQueueSize.GetAsInt()
wpConfig.Woodpecker.Client.SegmentRollingPolicy.MaxSize = cfg.SegmentRollingMaxSize.GetAsSize()
wpConfig.Woodpecker.Client.SegmentRollingPolicy.MaxInterval = cfg.SegmentRollingMaxTime.GetAsInt()
// logStore
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxInterval = cfg.SyncMaxInterval.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxEntries = cfg.SyncMaxEntries.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxBytes = cfg.SyncMaxBytes.GetAsSize()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushRetries = cfg.FlushMaxRetries.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushSize = cfg.FlushMaxSize.GetAsSize()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.MaxFlushThreads = cfg.FlushMaxThreads.GetAsInt()
wpConfig.Woodpecker.Logstore.LogFileSyncPolicy.RetryInterval = cfg.RetryInterval.GetAsInt()
wpConfig.Woodpecker.Logstore.FragmentManager.MaxBytes = cfg.FragmentCachedMaxBytes.GetAsSize()
wpConfig.Woodpecker.Logstore.FragmentManager.MaxInterval = cfg.FragmentCachedInterval.GetAsInt()
// storage
wpConfig.Woodpecker.Storage.Type = cfg.StorageType.GetValue()
wpConfig.Woodpecker.Storage.RootPath = cfg.RootPath.GetValue()
// set bucketName
wpConfig.Minio.BucketName = paramtable.Get().MinioCfg.BucketName.GetValue()
wpConfig.Minio.RootPath = paramtable.Get().MinioCfg.RootPath.GetValue()
// set log
wpConfig.Log.Level = paramtable.Get().LogCfg.Level.GetValue()
wpConfig.Log.Format = paramtable.Get().LogCfg.Format.GetValue()
wpConfig.Log.Stdout = paramtable.Get().LogCfg.Stdout.GetAsBool()
wpConfig.Log.File.RootPath = paramtable.Get().LogCfg.RootPath.GetValue()
wpConfig.Log.File.MaxSize = paramtable.Get().LogCfg.MaxSize.GetAsInt()
wpConfig.Log.File.MaxAge = paramtable.Get().LogCfg.MaxAge.GetAsInt()
wpConfig.Log.File.MaxBackups = paramtable.Get().LogCfg.MaxBackups.GetAsInt()
return nil
}
func (b *builderImpl) getMinioClient(ctx context.Context) (*minio.Client, error) {
c := objectstorage.NewDefaultConfig()
params := paramtable.Get()
opts := []objectstorage.Option{
objectstorage.RootPath(params.MinioCfg.RootPath.GetValue()),
objectstorage.Address(params.MinioCfg.Address.GetValue()),
objectstorage.AccessKeyID(params.MinioCfg.AccessKeyID.GetValue()),
objectstorage.SecretAccessKeyID(params.MinioCfg.SecretAccessKey.GetValue()),
objectstorage.UseSSL(params.MinioCfg.UseSSL.GetAsBool()),
objectstorage.SslCACert(params.MinioCfg.SslCACert.GetValue()),
objectstorage.BucketName(params.MinioCfg.BucketName.GetValue()),
objectstorage.UseIAM(params.MinioCfg.UseIAM.GetAsBool()),
objectstorage.CloudProvider(params.MinioCfg.CloudProvider.GetValue()),
objectstorage.IAMEndpoint(params.MinioCfg.IAMEndpoint.GetValue()),
objectstorage.UseVirtualHost(params.MinioCfg.UseVirtualHost.GetAsBool()),
objectstorage.Region(params.MinioCfg.Region.GetValue()),
objectstorage.RequestTimeout(params.MinioCfg.RequestTimeoutMs.GetAsInt64()),
objectstorage.CreateBucket(true),
objectstorage.GcpCredentialJSON(params.MinioCfg.GcpCredentialJSON.GetValue()),
}
for _, opt := range opts {
opt(c)
}
return objectstorage.NewMinioClient(ctx, c)
}
func (b *builderImpl) getEtcdClient(ctx context.Context) (*clientv3.Client, error) {
params := paramtable.Get()
etcdConfig := &params.EtcdCfg
log := log.Ctx(ctx)
etcdCli, err := etcd.CreateEtcdClient(
etcdConfig.UseEmbedEtcd.GetAsBool(),
etcdConfig.EtcdEnableAuth.GetAsBool(),
etcdConfig.EtcdAuthUserName.GetValue(),
etcdConfig.EtcdAuthPassword.GetValue(),
etcdConfig.EtcdUseSSL.GetAsBool(),
etcdConfig.Endpoints.GetAsStrings(),
etcdConfig.EtcdTLSCert.GetValue(),
etcdConfig.EtcdTLSKey.GetValue(),
etcdConfig.EtcdTLSCACert.GetValue(),
etcdConfig.EtcdTLSMinVersion.GetValue())
if err != nil {
log.Warn("Woodpecker walimpls connect to etcd failed", zap.Error(err))
return nil, err
}
return etcdCli, nil
}

86
pkg/streaming/walimpls/impls/wp/message_id.go Normal file
View File

@ -0,0 +1,86 @@
package wp
import (
"encoding/base64"
"fmt"
"github.com/cockroachdb/errors"
wp "github.com/zilliztech/woodpecker/woodpecker/log"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
)
func NewWpID(id *wp.LogMessageId) message.MessageID {
return wpID{
logMsgId: id,
}
}
func UnmarshalMessageID(data string) (message.MessageID, error) {
id, err := unmarshalMessageID(data)
if err != nil {
return nil, err
}
return id, nil
}
func unmarshalMessageID(data string) (wpID, error) {
val, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return wpID{logMsgId: nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when decode base64 with err: %s, id: %s", err.Error(), data)
}
msgID, err := wp.DeserializeLogMessageId(val)
if err != nil {
return wpID{logMsgId: nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when deserialize with err: %s, id: %s", err.Error(), data)
}
return wpID{logMsgId: msgID}, nil
}
var _ message.MessageID = wpID{}
type wpID struct {
logMsgId *wp.LogMessageId
}
func (id wpID) WoodpeckerID() *wp.LogMessageId {
return id.logMsgId
}
func (id wpID) WoodpeckerMsgId() *wp.LogMessageId {
return id.logMsgId
}
func (id wpID) WALName() string {
return walName
}
func (id wpID) LT(other message.MessageID) bool {
id2 := other.(wpID)
if id.logMsgId.SegmentId != id2.logMsgId.SegmentId {
return id.logMsgId.SegmentId < id2.logMsgId.SegmentId
}
return id.logMsgId.EntryId < id2.logMsgId.EntryId
}
func (id wpID) LTE(other message.MessageID) bool {
id2 := other.(wpID)
if id.logMsgId.SegmentId < id2.logMsgId.SegmentId {
return true
} else if id.logMsgId.SegmentId > id2.logMsgId.SegmentId {
return false
}
return id.logMsgId.EntryId <= id2.logMsgId.EntryId
}
func (id wpID) EQ(other message.MessageID) bool {
id2 := other.(wpID)
return id.logMsgId.SegmentId == id2.logMsgId.SegmentId && id.logMsgId.EntryId == id2.logMsgId.EntryId
}
func (id wpID) Marshal() string {
return base64.StdEncoding.EncodeToString(id.logMsgId.Serialize())
}
func (id wpID) String() string {
return fmt.Sprintf("%d/%d", id.logMsgId.SegmentId, id.logMsgId.EntryId)
}

64
pkg/streaming/walimpls/impls/wp/message_id_test.go Normal file
View File

@ -0,0 +1,64 @@
package wp
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
woodpecker "github.com/zilliztech/woodpecker/woodpecker/log"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
)
func TestMessageID(t *testing.T) {
wpId := message.MessageID(newMessageIDOfWoodpecker(1, 2)).(interface {
WoodpeckerID() *woodpecker.LogMessageId
}).WoodpeckerID()
assert.Equal(t, walName, newMessageIDOfWoodpecker(1, 2).WALName())
assert.Equal(t, int64(1), wpId.SegmentId)
assert.Equal(t, int64(2), wpId.EntryId)
ids := []wpID{
newMessageIDOfWoodpecker(0, 0),
newMessageIDOfWoodpecker(0, 1),
newMessageIDOfWoodpecker(0, 1000),
newMessageIDOfWoodpecker(1, 0),
newMessageIDOfWoodpecker(1, 1),
newMessageIDOfWoodpecker(1, 1000),
newMessageIDOfWoodpecker(2, 0),
newMessageIDOfWoodpecker(2, 1),
newMessageIDOfWoodpecker(2, 1000),
}
for x, idx := range ids {
for y, idy := range ids {
assert.Equal(t, idx.EQ(idy), x == y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idx.EQ(idy), idx, idy))
assert.Equal(t, idy.EQ(idx), x == y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idy.EQ(idx), idx, idy))
assert.Equal(t, idy.LT(idx), x > y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idy.LT(idx), idx, idy))
assert.Equal(t, idy.LTE(idx), x >= y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idy.LTE(idx), idx, idy))
assert.Equal(t, idx.LT(idy), x < y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idx.LT(idy), idx, idy))
assert.Equal(t, idx.LTE(idy), x <= y, fmt.Sprintf("expected %v when idx:%v,idy:%v", idx.LTE(idy), idx, idy))
}
}
msgID, err := UnmarshalMessageID(newMessageIDOfWoodpecker(1, 2).Marshal())
assert.NoError(t, err)
assert.True(t, msgID.EQ(newMessageIDOfWoodpecker(1, 2)))
_, err = UnmarshalMessageID(string([]byte{0x01, 0x02, 0x03, 0x04}))
assert.Error(t, err)
}
// newMessageIDOfWoodpecker only for test.
func newMessageIDOfWoodpecker(segmentID int64, entryID int64) wpID {
id := wpID{
logMsgId: &woodpecker.LogMessageId{
SegmentId: segmentID,
EntryId: entryID,
},
}
return id
}

58
pkg/streaming/walimpls/impls/wp/opener.go Normal file
View File

@ -0,0 +1,58 @@
package wp
import (
"context"
"github.com/zilliztech/woodpecker/woodpecker"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/v2/log"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/helper"
)
var _ walimpls.OpenerImpls = (*openerImpl)(nil)
// openerImpl is the implementation of walimpls.Opener interface.
type openerImpl struct {
c woodpecker.Client
}
// Open opens a new wal.
func (o *openerImpl) Open(ctx context.Context, opt *walimpls.OpenOption) (walimpls.WALImpls, error) {
exists, err := o.c.LogExists(ctx, opt.Channel.Name)
if err != nil {
log.Ctx(ctx).Error("failed to check log exists", zap.String("log_name", opt.Channel.Name), zap.Error(err))
return nil, err
}
if !exists {
if err := o.c.CreateLog(ctx, opt.Channel.Name); err != nil {
log.Ctx(ctx).Error("failed to create log", zap.String("log_name", opt.Channel.Name), zap.Error(err))
return nil, err
}
}
l, err := o.c.OpenLog(ctx, opt.Channel.Name)
if err != nil {
log.Ctx(ctx).Error("failed to open log", zap.String("log_name", opt.Channel.Name), zap.Error(err))
return nil, err
}
p, err := l.OpenLogWriter(ctx)
if err != nil {
log.Ctx(ctx).Error("failed to open log writer", zap.String("log_name", opt.Channel.Name), zap.Error(err))
return nil, err
}
log.Ctx(ctx).Info("finish to open log writer", zap.String("log_name", opt.Channel.Name), zap.Error(err))
return &walImpl{
WALHelper: helper.NewWALHelper(opt),
p: p,
l: l,
}, nil
}
// Close closes the opener resources.
func (o *openerImpl) Close() {
err := o.c.Close()
if err != nil {
log.Ctx(context.Background()).Error("failed to close woodpecker client", zap.Error(err))
}
}

78
pkg/streaming/walimpls/impls/wp/scanner.go Normal file
View File

@ -0,0 +1,78 @@
package wp
import (
"context"
"github.com/cockroachdb/errors"
woodpecker "github.com/zilliztech/woodpecker/woodpecker/log"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/v2/log"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/helper"
)
var _ walimpls.ScannerImpls = (*scannerImpl)(nil)
type scannerImpl struct {
*helper.ScannerHelper
reader woodpecker.LogReader
msgChannel chan message.ImmutableMessage
}
func newScanner(scannerName string, reader woodpecker.LogReader) *scannerImpl {
s := &scannerImpl{
ScannerHelper: helper.NewScannerHelper(scannerName),
reader: reader,
msgChannel: make(chan message.ImmutableMessage, 1),
}
go s.executeConsumer()
return s
}
func (s *scannerImpl) Chan() <-chan message.ImmutableMessage {
return s.msgChannel
}
func (s *scannerImpl) Close() error {
err := s.ScannerHelper.Close()
if err != nil {
log.Ctx(s.Context()).Warn("failed to close wp scanner", zap.Error(err))
}
if s.reader != nil {
err = s.reader.Close(context.Background())
if err != nil {
log.Ctx(s.Context()).Warn("failed to close wp reader", zap.Error(err))
}
}
return err
}
func (s *scannerImpl) executeConsumer() {
defer close(s.msgChannel)
for {
msg, err := s.reader.ReadNext(s.Context())
if err != nil {
if errors.IsAny(err, context.Canceled, context.DeadlineExceeded) {
s.Finish(nil)
return
}
log.Ctx(s.Context()).Error("wp readNext msg exception", zap.Error(err))
s.Finish(err)
return
}
newImmutableMessage := message.NewImmutableMesasge(
wpID{msg.Id},
msg.Payload,
msg.Properties,
)
select {
case <-s.Context().Done():
s.Finish(nil)
return
case s.msgChannel <- newImmutableMessage:
}
}
}

77
pkg/streaming/walimpls/impls/wp/wal.go Normal file
View File

@ -0,0 +1,77 @@
package wp
import (
"context"
wp "github.com/zilliztech/woodpecker/woodpecker/log"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/v2/proto/streamingpb"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/helper"
)
var _ walimpls.WALImpls = (*walImpl)(nil)
type walImpl struct {
*helper.WALHelper
p wp.LogWriter
l wp.LogHandle
}
func (w *walImpl) WALName() string {
return walName
}
func (w *walImpl) Append(ctx context.Context, msg message.MutableMessage) (message.MessageID, error) {
r := w.p.Write(ctx,
&wp.WriterMessage{
Payload: msg.Payload(),
Properties: msg.Properties().ToRawMap(),
},
)
if r.Err != nil {
w.Log().RatedWarn(1, "write message to woodpecker failed", zap.Error(r.Err))
return nil, r.Err
}
return wpID{r.LogMessageId}, nil
}
func (w *walImpl) Read(ctx context.Context, opt walimpls.ReadOption) (walimpls.ScannerImpls, error) {
from := wp.LatestLogMessageID()
switch t := opt.DeliverPolicy.GetPolicy().(type) {
case *streamingpb.DeliverPolicy_All:
from = wp.EarliestLogMessageID()
case *streamingpb.DeliverPolicy_Latest:
from = wp.LatestLogMessageID()
case *streamingpb.DeliverPolicy_StartFrom:
id, err := unmarshalMessageID(t.StartFrom.GetId())
if err != nil {
return nil, err
}
from.SegmentId = id.logMsgId.SegmentId
from.EntryId = id.logMsgId.EntryId
case *streamingpb.DeliverPolicy_StartAfter:
id, err := unmarshalMessageID(t.StartAfter.GetId())
if err != nil {
return nil, err
}
from.SegmentId = id.logMsgId.SegmentId
from.EntryId = id.logMsgId.EntryId + 1
}
reader, err := w.l.OpenLogReader(ctx, &from)
if err != nil {
return nil, err
}
return newScanner(opt.Name, reader), nil
}
func (w *walImpl) Close() {
closeWriterErr := w.p.Close(context.Background())
if closeWriterErr != nil {
w.Log().Warn("close woodpecker writer err", zap.Error(closeWriterErr))
}
}

31
pkg/streaming/walimpls/impls/wp/wp_test.go Normal file
View File

@ -0,0 +1,31 @@
package wp
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls"
"github.com/milvus-io/milvus/pkg/v2/streaming/walimpls/registry"
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
)
func TestMain(m *testing.M) {
paramtable.Init()
m.Run()
}
func TestRegistry(t *testing.T) {
registeredB := registry.MustGetBuilder(walName)
assert.NotNil(t, registeredB)
assert.Equal(t, walName, registeredB.Name())
id, err := message.UnmarshalMessageID(walName, newMessageIDOfWoodpecker(1, 2).Marshal())
assert.NoError(t, err)
assert.True(t, id.EQ(newMessageIDOfWoodpecker(1, 2)))
}
func TestWAL(t *testing.T) {
walimpls.NewWALImplsTestFramework(t, 100, &builderImpl{}).Run()
}

202
pkg/util/paramtable/service_param.go
View File

@ -46,6 +46,7 @@ type ServiceParam struct {
EtcdCfg EtcdConfig
TiKVCfg TiKVConfig
MQCfg MQConfig
WoodpeckerCfg WoodpeckerConfig
PulsarCfg PulsarConfig
KafkaCfg KafkaConfig
RocksmqCfg RocksmqConfig
@ -60,6 +61,7 @@ func (p *ServiceParam) init(bt *BaseTable) {
p.EtcdCfg.Init(bt)
p.TiKVCfg.Init(bt)
p.MQCfg.Init(bt)
p.WoodpeckerCfg.Init(bt)
p.PulsarCfg.Init(bt)
p.KafkaCfg.Init(bt)
p.RocksmqCfg.Init(bt)
@ -85,6 +87,10 @@ func (p *ServiceParam) KafkaEnable() bool {
return p.KafkaCfg.Address.GetValue() != ""
}
func (p *ServiceParam) WoodpeckerEnable() bool {
return p.WoodpeckerCfg.MetaPrefix.GetValue() != ""
}
// /////////////////////////////////////////////////////////////////////////////
// --- etcd ---
type EtcdConfig struct {
@ -542,7 +548,7 @@ func (p *MQConfig) Init(base *BaseTable) {
Version: "2.3.0",
DefaultValue: "default",
Doc: `Default value: "default"
Valid values: [default, pulsar, kafka, rocksmq, natsmq]`,
Valid values: [default, pulsar, kafka, rocksmq, natsmq, woodpecker]`,
Export: true,
}
p.Type.Init(base.mgr)
@ -644,6 +650,200 @@ Valid values: [default, pulsar, kafka, rocksmq, natsmq]`,
p.IgnoreBadPosition.Init(base.mgr)
}
// /////////////////////////////////////////////////////////////////////////////
// --- woodpecker ---
type WoodpeckerConfig struct {
// meta
MetaType ParamItem `refreshable:"false"`
MetaPrefix ParamItem `refreshable:"false"`
// client
AppendQueueSize ParamItem `refreshable:"true"`
AppendMaxRetries ParamItem `refreshable:"true"`
SegmentRollingMaxSize ParamItem `refreshable:"true"`
SegmentRollingMaxTime ParamItem `refreshable:"true"`
AuditorMaxInterval ParamItem `refreshable:"true"`
// logstore
SyncMaxInterval ParamItem `refreshable:"true"`
SyncMaxBytes ParamItem `refreshable:"true"`
SyncMaxEntries ParamItem `refreshable:"true"`
FlushMaxRetries ParamItem `refreshable:"true"`
RetryInterval ParamItem `refreshable:"true"`
FlushMaxSize ParamItem `refreshable:"true"`
FlushMaxThreads ParamItem `refreshable:"true"`
FragmentCachedMaxBytes ParamItem `refreshable:"true"`
FragmentCachedInterval ParamItem `refreshable:"true"`
// storage
StorageType ParamItem `refreshable:"false"`
RootPath ParamItem `refreshable:"false"`
}
func (p *WoodpeckerConfig) Init(base *BaseTable) {
p.MetaType = ParamItem{
Key: "woodpecker.meta.type",
Version: "2.6.0",
DefaultValue: "etcd",
Doc: "The Type of the metadata provider. currently only support etcd.",
Export: true,
}
p.MetaType.Init(base.mgr)
p.MetaPrefix = ParamItem{
Key: "woodpecker.meta.prefix",
Version: "2.6.0",
DefaultValue: "woodpecker",
Doc: "The Prefix of the metadata provider. default is woodpecker.",
Export: true,
}
p.MetaPrefix.Init(base.mgr)
p.AppendQueueSize = ParamItem{
Key: "woodpecker.client.segmentAppend.queueSize",
Version: "2.6.0",
DefaultValue: "10000",
Doc: "The size of the queue for pending messages to be sent of each log.",
Export: true,
}
p.AppendQueueSize.Init(base.mgr)
p.AppendMaxRetries = ParamItem{
Key: "woodpecker.client.segmentAppend.maxRetries",
Version: "2.6.0",
DefaultValue: "3",
Doc: "Maximum number of retries for segment append operations.",
Export: true,
}
p.AppendMaxRetries.Init(base.mgr)
p.SegmentRollingMaxSize = ParamItem{
Key: "woodpecker.client.segmentRollingPolicy.maxSize",
Version: "2.6.0",
DefaultValue: "2000000000", // 1 GB
Doc: "Maximum entries count of a segment, default is 2GB",
Export: true,
}
p.SegmentRollingMaxSize.Init(base.mgr)
p.SegmentRollingMaxTime = ParamItem{
Key: "woodpecker.client.segmentRollingPolicy.maxInterval",
Version: "2.6.0",
DefaultValue: "600",
Doc: "Maximum interval between two segments in seconds, default is 10 minutes.",
Export: true,
}
p.SegmentRollingMaxTime.Init(base.mgr)
p.AuditorMaxInterval = ParamItem{
Key: "woodpecker.client.auditor.maxInterval",
Version: "2.6.0",
DefaultValue: "10",
Doc: "Maximum interval between two auditing operations in seconds, default is 10 seconds.",
Export: true,
}
p.AuditorMaxInterval.Init(base.mgr)
p.SyncMaxInterval = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxInterval",
Version: "2.6.0",
DefaultValue: "1000",
Doc: "Maximum interval between two sync operations in milliseconds.",
Export: true,
}
p.SyncMaxInterval.Init(base.mgr)
p.SyncMaxEntries = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxEntries",
Version: "2.6.0",
DefaultValue: "100000",
Doc: "Maximum entries number of write buffer.",
Export: true,
}
p.SyncMaxEntries.Init(base.mgr)
p.SyncMaxBytes = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxBytes",
Version: "2.6.0",
DefaultValue: "64000000",
Doc: "Maximum size of write buffer in bytes.",
Export: true,
}
p.SyncMaxBytes.Init(base.mgr)
p.FlushMaxRetries = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushRetries",
Version: "2.6.0",
DefaultValue: "5",
Doc: "Maximum size of write buffer in bytes.",
Export: true,
}
p.FlushMaxRetries.Init(base.mgr)
p.FlushMaxSize = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushSize",
Version: "2.6.0",
DefaultValue: "8000000",
Doc: "Maximum size of a fragment in bytes to flush, default is 8M.",
Export: true,
}
p.FlushMaxSize.Init(base.mgr)
p.RetryInterval = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.retryInterval",
Version: "2.6.0",
DefaultValue: "1000",
Doc: "Maximum interval between two retries in milliseconds.",
Export: true,
}
p.RetryInterval.Init(base.mgr)
p.FlushMaxThreads = ParamItem{
Key: "woodpecker.logstore.logFileSyncPolicy.maxFlushThreads",
Version: "2.6.0",
DefaultValue: "4",
Doc: "Maximum number of threads to flush data",
Export: true,
}
p.FlushMaxThreads.Init(base.mgr)
p.FragmentCachedMaxBytes = ParamItem{
Key: "woodpecker.logstore.fragmentManager.maxBytes",
Version: "2.6.0",
DefaultValue: "1000000000",
Doc: "Maximum size of fragment cached data in bytes.",
Export: true,
}
p.FragmentCachedMaxBytes.Init(base.mgr)
p.FragmentCachedInterval = ParamItem{
Key: "woodpecker.logstore.fragmentManager.maxInterval",
Version: "2.6.0",
DefaultValue: "1000",
Doc: "Maximum interval between two fragment evicts in milliseconds.",
Export: true,
}
p.FragmentCachedInterval.Init(base.mgr)
p.StorageType = ParamItem{
Key: "woodpecker.storage.type",
Version: "2.6.0",
DefaultValue: "minio",
Doc: "The Type of the storage provider. Valid values: [default, minio, local, service], default is minio.",
Export: true,
}
p.StorageType.Init(base.mgr)
p.RootPath = ParamItem{
Key: "woodpecker.storage.rootPath",
Version: "2.6.0",
DefaultValue: "/var/lib/milvus/woodpecker",
Doc: "The root path of the storage provider.",
Export: true,
}
p.RootPath.Init(base.mgr)
}
// /////////////////////////////////////////////////////////////////////////////
// --- pulsar ---
type PulsarConfig struct {

25
pkg/util/paramtable/service_param_test.go
View File

@ -96,6 +96,31 @@ func TestServiceParam(t *testing.T) {
SParams.init(bt)
})
t.Run("test woodpeckerConfig", func(t *testing.T) {
wpCfg := &SParams.WoodpeckerCfg
assert.Equal(t, wpCfg.MetaType.GetValue(), "etcd")
assert.Equal(t, wpCfg.MetaPrefix.GetValue(), "woodpecker")
assert.Equal(t, wpCfg.AppendQueueSize.GetAsInt(), 10000)
assert.Equal(t, wpCfg.AppendMaxRetries.GetAsInt(), 3)
assert.Equal(t, wpCfg.SegmentRollingMaxSize.GetAsInt(), 2000000000)
assert.Equal(t, wpCfg.SegmentRollingMaxTime.GetAsInt(), 600)
assert.Equal(t, wpCfg.AuditorMaxInterval.GetAsInt(), 10)
assert.Equal(t, wpCfg.SyncMaxInterval.GetAsInt(), 1000)
assert.Equal(t, wpCfg.SyncMaxEntries.GetAsInt(), 100000)
assert.Equal(t, wpCfg.SyncMaxBytes.GetAsInt(), 64000000)
assert.Equal(t, wpCfg.FlushMaxRetries.GetAsInt(), 5)
assert.Equal(t, wpCfg.FlushMaxSize.GetAsInt(), 8000000)
assert.Equal(t, wpCfg.FlushMaxThreads.GetAsInt(), 4)
assert.Equal(t, wpCfg.RetryInterval.GetAsInt(), 1000)
assert.Equal(t, wpCfg.FragmentCachedMaxBytes.GetAsInt64(), int64(1000000000))
assert.Equal(t, wpCfg.FragmentCachedInterval.GetAsInt(), 1000)
assert.Equal(t, wpCfg.StorageType.GetValue(), "minio")
assert.Equal(t, wpCfg.RootPath.GetValue(), "/var/lib/milvus/woodpecker")
})
t.Run("test pulsarConfig", func(t *testing.T) {
// test default value
{