mirror of https://github.com/milvus-io/milvus.git
Delete unused code
Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>pull/4973/head^2
zhenshan.cao
yefu.chen
parent
c732c267e1
commit
9c490a209b
|
|
@ -1,13 +1,14 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"go.uber.org/zap"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"context"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proxy"
|
||||
|
|
@ -59,4 +60,4 @@ func main() {
|
|||
|
||||
func exit(code int) {
|
||||
os.Exit(code)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
7
go.mod
7
go.mod
|
|
@ -5,7 +5,6 @@ go 1.15
|
|||
require (
|
||||
code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 // indirect
|
||||
github.com/apache/pulsar-client-go v0.1.1
|
||||
github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af
|
||||
github.com/aws/aws-sdk-go v1.30.8
|
||||
github.com/coreos/etcd v3.3.25+incompatible // indirect
|
||||
github.com/frankban/quicktest v1.10.2 // indirect
|
||||
|
|
@ -22,9 +21,9 @@ require (
|
|||
github.com/onsi/gomega v1.10.0 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible // indirect
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 // indirect
|
||||
github.com/pingcap/errors v0.11.4 // indirect
|
||||
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463
|
||||
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 // indirect
|
||||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48
|
||||
github.com/prometheus/client_golang v1.5.1 // indirect
|
||||
github.com/prometheus/common v0.10.0 // indirect
|
||||
|
|
@ -34,10 +33,8 @@ require (
|
|||
github.com/spaolacci/murmur3 v1.1.0
|
||||
github.com/stretchr/testify v1.6.1
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b
|
||||
github.com/tikv/pd v2.1.19+incompatible
|
||||
github.com/yahoo/athenz v1.9.16 // indirect
|
||||
go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738
|
||||
go.uber.org/atomic v1.6.0
|
||||
go.uber.org/zap v1.15.0
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b // indirect
|
||||
|
|
|
|||
31
go.sum
31
go.sum
|
|
@ -8,17 +8,13 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
|
|||
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
|
||||
github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
|
||||
github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af h1:hE//BVq7iKVPkKwA9ZHGUF8DY/DhkzBGN7LGDmUpzC8=
|
||||
github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af/go.mod h1:QdYxU2iG99VVU6cvoBRkCgkazfJSL9WwPZ20PZR6aUk=
|
||||
github.com/ardielle/ardielle-go v1.5.2 h1:TilHTpHIQJ27R1Tl/iITBzMwiUGSlVfiVhwDNGM3Zj4=
|
||||
github.com/ardielle/ardielle-go v1.5.2/go.mod h1:I4hy1n795cUhaVt/ojz83SNVCYIGsAFAONtv2Dr7HUI=
|
||||
github.com/ardielle/ardielle-tools v1.5.4/go.mod h1:oZN+JRMnqGiIhrzkRN9l26Cej9dEx4jeNG6A+AdkShk=
|
||||
github.com/aws/aws-sdk-go v1.30.8 h1:4BHbh8K3qKmcnAgToZ2LShldRF9inoqIBccpCLNCy3I=
|
||||
github.com/aws/aws-sdk-go v1.30.8/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
|
||||
github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHLutezwnBvyneBbwvB8C82y3dcoOj5EQJIdGXA=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
|
||||
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
|
||||
|
|
@ -60,7 +56,6 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
|
|||
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
|
||||
github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d844Tk=
|
||||
github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
|
|
@ -106,7 +101,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
|||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/mux v1.7.0 h1:tOSd0UKHQd6urX6ApfOn4XdBMY6Sh1MfxV3kmaazO+U=
|
||||
github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
|
||||
github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
|
||||
|
|
@ -121,7 +115,6 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
|
|||
github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
|
||||
github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk=
|
||||
|
|
@ -142,7 +135,6 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
|
|||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.10.8 h1:eLeJ3dr/Y9+XRfJT4l+8ZjmtB5RPJhucH2HeCV5+IZY=
|
||||
github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
github.com/klauspost/compress v1.10.11 h1:K9z59aO18Aywg2b/WSgBaUX99mHy2BES18Cr5lBKZHk=
|
||||
github.com/klauspost/compress v1.10.11/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
|
||||
|
|
@ -153,18 +145,14 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
|
|||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY=
|
||||
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
|
|
@ -193,11 +181,9 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
|
|||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
|
||||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1 h1:mFwc4LvZ0xpSvDZ3E+k8Yte0hLOMxXUlP+yXtJqkYfQ=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.0 h1:Gwkk+PTu/nfOwNMtUB/mRUv0X7ewW5dO4AERT1ThVKo=
|
||||
|
|
@ -207,7 +193,6 @@ github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+
|
|||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible h1:CAG0PUvo1fen+ZEfxKJjFIc8GuuN5RuaBuCAuaP2Hno=
|
||||
github.com/ozonru/etcd v3.3.20-grpc1.27-origmodule+incompatible/go.mod h1:iIubILNIN6Jq9h8uiSLrN9L1tuj3iSSFwz3R61skm/A=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI=
|
||||
github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
|
|
@ -231,7 +216,6 @@ github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZ
|
|||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:2JCf+JCLBs7IUZzYdIrSDN+GWYacKOdToIAt5zcga54=
|
||||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:43j3yLP9UiXa0z95/W3hN7yTjoxsQoOll5rrGBgBcnE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
|
|
@ -239,25 +223,21 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
|||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
|
||||
github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
|
||||
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
|
||||
github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
|
||||
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
|
||||
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
|
||||
|
|
@ -275,7 +255,6 @@ github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I
|
|||
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
|
|
@ -285,7 +264,6 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO
|
|||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
|
||||
github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4=
|
||||
github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
||||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
|
|
@ -301,8 +279,6 @@ github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd
|
|||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc=
|
||||
github.com/tikv/pd v2.1.19+incompatible h1:rqjHqO7t/STke/R2Yz6+lQj6NPA8u7G2Otwqup4K+P8=
|
||||
github.com/tikv/pd v2.1.19+incompatible/go.mod h1:v6C/D7ONC49SgjI4jbGnooSizvijaO/bdIm62DVR4tI=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
|
|
@ -315,7 +291,6 @@ github.com/valyala/gozstd v1.7.0 h1:Ljh5c9zboqLhwTI33al32R72iCZfn0mCbVGcFWbGwRQ=
|
|||
github.com/valyala/gozstd v1.7.0/go.mod h1:y5Ew47GLlP37EkTB+B4s7r6A5rdaeB7ftbl9zoYiIPQ=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/yahoo/athenz v1.8.55 h1:xGhxN3yLq334APyn0Zvcc+aqu78Q7BBhYJevM3EtTW0=
|
||||
github.com/yahoo/athenz v1.8.55/go.mod h1:G7LLFUH7Z/r4QAB7FfudfuA7Am/eCzO1GlzBhDL6Kv0=
|
||||
github.com/yahoo/athenz v1.9.16 h1:2s8KtIxwAbcJIYySsfrT/t/WO0Ss5O7BPGUN/q8x2bg=
|
||||
github.com/yahoo/athenz v1.9.16/go.mod h1:guj+0Ut6F33wj+OcSRlw69O0itsR7tVocv15F2wJnIo=
|
||||
|
|
@ -346,7 +321,6 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a h1:vclmkQCjlDX5OydZ9wv8rBCcS0QyQY66Mpf/7BZbInM=
|
||||
golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
|
|
@ -442,19 +416,15 @@ google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg=
|
|||
google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/linkedin/goavro.v1 v1.0.5 h1:BJa69CDh0awSsLUmZ9+BowBdokpduDZSM9Zk8oKHfN4=
|
||||
gopkg.in/linkedin/goavro.v1 v1.0.5/go.mod h1:Aw5GdAbizjOEl0kAMHV9iHmA8reZzW/OKuJAl4Hb9F0=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
|
|
@ -467,7 +437,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
|||
|
|
@ -25,8 +25,6 @@ type GlobalIdAllocator struct {
|
|||
allocator tso.Allocator
|
||||
}
|
||||
|
||||
//func getID
|
||||
|
||||
var allocator GlobalIdAllocator = GlobalIdAllocator{
|
||||
allocator: tso.NewGlobalTSOAllocator("idTimestamp"),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import (
|
|||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
pb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
type UniqueID = typeutil.UniqueID
|
||||
|
|
@ -38,6 +38,7 @@ type TimeTickReader struct {
|
|||
func (r *TimeTickReader) Start() {
|
||||
go r.readTimeTick()
|
||||
go r.timeSync()
|
||||
|
||||
}
|
||||
|
||||
func (r *TimeTickReader) Close() {
|
||||
|
|
@ -73,7 +74,10 @@ func (r *TimeTickReader) timeSync() {
|
|||
}
|
||||
}
|
||||
//send timestamp flag to reader channel
|
||||
msg := pb.InsertOrDeleteMsg{Timestamp: minTimeStamp, ClientId: stopReadFlagId}
|
||||
msg := internalpb.TimeTickMsg{
|
||||
Timestamp: minTimeStamp,
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
}
|
||||
payload, err := proto.Marshal(&msg)
|
||||
if err != nil {
|
||||
//TODO log error
|
||||
|
|
@ -101,12 +105,12 @@ func (r *TimeTickReader) readTimeTick() {
|
|||
}
|
||||
|
||||
msg := cm.Message
|
||||
var tsm pb.TimeSyncMsg
|
||||
var tsm internalpb.TimeTickMsg
|
||||
if err := proto.Unmarshal(msg.Payload(), &tsm); err != nil {
|
||||
log.Printf("UnMarshal timetick flag error %v", err)
|
||||
}
|
||||
|
||||
r.timeTickPeerProxy[tsm.Peer_Id] = tsm.Timestamp
|
||||
r.timeTickPeerProxy[tsm.PeerId] = tsm.Timestamp
|
||||
r.timeTickConsumer.AckID(msg.ID())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,15 +14,18 @@
|
|||
package tso
|
||||
|
||||
import (
|
||||
"log"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/conf"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
|
||||
"github.com/pingcap/log"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -66,7 +69,7 @@ func NewGlobalTSOAllocator(key string) Allocator {
|
|||
var saveInterval time.Duration = 3 * time.Second
|
||||
return &GlobalTSOAllocator{
|
||||
timestampOracle: ×tampOracle{
|
||||
client: client,
|
||||
kvBase: kv.NewEtcdKV(client, path.Join(conf.Config.Etcd.Rootpath, "tso")),
|
||||
rootPath: conf.Config.Etcd.Rootpath,
|
||||
saveInterval: saveInterval,
|
||||
maxResetTSGap: func() time.Duration { return 3 * time.Second },
|
||||
|
|
@ -104,7 +107,7 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (uint64, error) {
|
|||
current := (*atomicObject)(atomic.LoadPointer(>a.timestampOracle.TSO))
|
||||
if current == nil || current.physical == typeutil.ZeroTime {
|
||||
// If it's leader, maybe SyncTimestamp hasn't completed yet
|
||||
log.Info("sync hasn't completed yet, wait for a while")
|
||||
log.Println("sync hasn't completed yet, wait for a while")
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
|
@ -112,7 +115,7 @@ func (gta *GlobalTSOAllocator) GenerateTSO(count uint32) (uint64, error) {
|
|||
physical = current.physical.UnixNano() / int64(time.Millisecond)
|
||||
logical = atomic.AddInt64(¤t.logical, int64(count))
|
||||
if logical >= maxLogical {
|
||||
log.Error("logical part outside of max logical interval, please check ntp time",
|
||||
log.Println("logical part outside of max logical interval, please check ntp time",
|
||||
zap.Int("retry-count", i))
|
||||
time.Sleep(UpdateTimestampStep)
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ package tso
|
|||
|
||||
import (
|
||||
"log"
|
||||
"path"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
|
@ -23,10 +22,9 @@ import (
|
|||
"go.uber.org/zap"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/etcdutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/kv"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/tsoutil"
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -48,9 +46,10 @@ type atomicObject struct {
|
|||
|
||||
// timestampOracle is used to maintain the logic of tso.
|
||||
type timestampOracle struct {
|
||||
client *clientv3.Client
|
||||
rootPath string
|
||||
key string
|
||||
kvBase *kv.EtcdKV
|
||||
|
||||
// TODO: remove saveInterval
|
||||
saveInterval time.Duration
|
||||
maxResetTSGap func() time.Duration
|
||||
|
|
@ -59,40 +58,29 @@ type timestampOracle struct {
|
|||
lastSavedTime atomic.Value
|
||||
}
|
||||
|
||||
func (t *timestampOracle) getTimestampPath() string {
|
||||
return path.Join(t.rootPath, t.key)
|
||||
}
|
||||
|
||||
func (t *timestampOracle) loadTimestamp() (time.Time, error) {
|
||||
data, err := etcdutil.GetValue(t.client, t.getTimestampPath())
|
||||
strData, err := t.kvBase.Load(t.key)
|
||||
|
||||
var binData []byte = []byte(strData)
|
||||
|
||||
if err != nil {
|
||||
return typeutil.ZeroTime, err
|
||||
}
|
||||
if len(data) == 0 {
|
||||
if len(binData) == 0 {
|
||||
return typeutil.ZeroTime, nil
|
||||
}
|
||||
return typeutil.ParseTimestamp(data)
|
||||
return typeutil.ParseTimestamp(binData)
|
||||
}
|
||||
|
||||
// save timestamp, if lastTs is 0, we think the timestamp doesn't exist, so create it,
|
||||
// otherwise, update it.
|
||||
func (t *timestampOracle) saveTimestamp(ts time.Time) error {
|
||||
key := t.getTimestampPath()
|
||||
data := typeutil.Uint64ToBytes(uint64(ts.UnixNano()))
|
||||
err := errors.New("")
|
||||
println("%v,%v", key, data)
|
||||
//resp, err := leadership.LeaderTxn().
|
||||
// Then(clientv3.OpPut(key, string(data))).
|
||||
// Commit()
|
||||
err := t.kvBase.Save(t.key, string(data))
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
//if !resp.Succeeded {
|
||||
// return errors.New("save timestamp failed, maybe we lost leader")
|
||||
//}
|
||||
|
||||
t.lastSavedTime.Store(ts)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ func getTsMsg(msgType MsgType, reqId UniqueID, hashValue int32) *TsMsg {
|
|||
tsMsg = searchMsg
|
||||
case internalPb.MsgType_kSearchResult:
|
||||
searchResult := internalPb.SearchResult{
|
||||
MsgType: internalPb.MsgType_kSearchResult,
|
||||
MsgType: internalPb.MsgType_kSearchResult,
|
||||
Status: &commonPb.Status{ErrorCode: commonPb.ErrorCode_SUCCESS},
|
||||
ReqId: reqId,
|
||||
ProxyId: 1,
|
||||
|
|
@ -93,7 +93,7 @@ func getTsMsg(msgType MsgType, reqId UniqueID, hashValue int32) *TsMsg {
|
|||
tsMsg = searchResultMsg
|
||||
case internalPb.MsgType_kTimeTick:
|
||||
timeTickResult := internalPb.TimeTickMsg{
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
MsgType: internalPb.MsgType_kTimeTick,
|
||||
PeerId: reqId,
|
||||
Timestamp: 1,
|
||||
}
|
||||
|
|
@ -175,8 +175,7 @@ func TestStream_Insert(t *testing.T) {
|
|||
initStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName, &msgPack, internalPb.MsgType_kInsert, internalPb.MsgType_kInsert, false)
|
||||
}
|
||||
|
||||
func
|
||||
TestStream_Delete(t *testing.T) {
|
||||
func TestStream_Delete(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"delete"}
|
||||
consumerChannels := []string{"delete"}
|
||||
|
|
@ -204,8 +203,7 @@ func TestStream_Search(t *testing.T) {
|
|||
initStream(pulsarAddress, producerChannels, consumerChannels, consumerSubName, &msgPack, internalPb.MsgType_kSearch, internalPb.MsgType_kSearch, false)
|
||||
}
|
||||
|
||||
func
|
||||
TestStream_SearchResult(t *testing.T) {
|
||||
func TestStream_SearchResult(t *testing.T) {
|
||||
pulsarAddress := "pulsar://localhost:6650"
|
||||
producerChannels := []string{"search"}
|
||||
consumerChannels := []string{"search"}
|
||||
|
|
|
|||
|
|
@ -217,7 +217,7 @@ func TestNewTtStream_Insert_TimeSync(t *testing.T) {
|
|||
Timestamps: []Timestamp{1},
|
||||
}
|
||||
insertMsg := &InsertMsg{
|
||||
BaseMsg: BaseMsg{HashValues: []int32{2}},
|
||||
BaseMsg: BaseMsg{HashValues: []int32{2}},
|
||||
InsertRequest: insertRequest,
|
||||
}
|
||||
var tsMsg TsMsg = insertMsg
|
||||
|
|
@ -245,7 +245,7 @@ func TestNewTtStream_Insert_TimeSync(t *testing.T) {
|
|||
fmt.Println("msg type: ", (*v).Type(), ", msg value: ", *v)
|
||||
}
|
||||
}
|
||||
if receiveCount + 1 >= len(msgPack.Msgs) {
|
||||
if receiveCount+1 >= len(msgPack.Msgs) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,11 +15,11 @@ type TsMsg interface {
|
|||
|
||||
type BaseMsg struct {
|
||||
BeginTimestamp Timestamp
|
||||
EndTimestamp Timestamp
|
||||
HashValues []int32
|
||||
EndTimestamp Timestamp
|
||||
HashValues []int32
|
||||
}
|
||||
|
||||
func (bm *BaseMsg) BeginTs() Timestamp{
|
||||
func (bm *BaseMsg) BeginTs() Timestamp {
|
||||
return bm.BeginTimestamp
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ package proxy
|
|||
|
||||
import (
|
||||
"context"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/pingcap/goleveldb/leveldb/errors"
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
|
||||
|
|
|
|||
|
|
@ -3,12 +3,13 @@ package proxy
|
|||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/msgstream"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/masterpb"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
|
||||
"log"
|
||||
)
|
||||
|
||||
type task interface {
|
||||
|
|
@ -96,10 +97,10 @@ func (it *InsertTask) Notify(err error) {
|
|||
type CreateCollectionTask struct {
|
||||
internalpb.CreateCollectionRequest
|
||||
masterClient masterpb.MasterClient
|
||||
done chan error
|
||||
resultChan chan *commonpb.Status
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
done chan error
|
||||
resultChan chan *commonpb.Status
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
}
|
||||
|
||||
func (cct *CreateCollectionTask) Id() UniqueID {
|
||||
|
|
@ -132,7 +133,7 @@ func (cct *CreateCollectionTask) Execute() error {
|
|||
log.Printf("create collection failed, error= %v", err)
|
||||
cct.resultChan <- &commonpb.Status{
|
||||
ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR,
|
||||
Reason: err.Error(),
|
||||
Reason: err.Error(),
|
||||
}
|
||||
} else {
|
||||
cct.resultChan <- resp
|
||||
|
|
@ -148,9 +149,9 @@ func (cct *CreateCollectionTask) WaitToFinish() error {
|
|||
defer cct.cancel()
|
||||
for {
|
||||
select {
|
||||
case err := <- cct.done:
|
||||
case err := <-cct.done:
|
||||
return err
|
||||
case <- cct.ctx.Done():
|
||||
case <-cct.ctx.Done():
|
||||
log.Print("wait to finish failed, timeout!")
|
||||
return errors.New("wait to finish failed, timeout!")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ import (
|
|||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
pb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
)
|
||||
|
||||
type timeTick struct {
|
||||
|
|
@ -33,10 +34,10 @@ func (tt *timeTick) tick() error {
|
|||
if tt.areRequestsDelivered(tt.currentTick) == false {
|
||||
return errors.New("Failed")
|
||||
}
|
||||
tsm := pb.TimeSyncMsg{
|
||||
tsm := internalpb.TimeTickMsg{
|
||||
MsgType: internalpb.MsgType_kTimeTick,
|
||||
PeerId: tt.peer_id,
|
||||
Timestamp: uint64(tt.currentTick),
|
||||
Peer_Id: tt.peer_id,
|
||||
SyncType: pb.SyncType_READ,
|
||||
}
|
||||
payload, err := proto.Marshal(&tsm)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
pb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
||||
)
|
||||
|
||||
func TestTimeTick(t *testing.T) {
|
||||
|
|
@ -72,7 +72,7 @@ func TestTimeTick(t *testing.T) {
|
|||
return
|
||||
}
|
||||
msg := cm.Message
|
||||
var tsm pb.TimeSyncMsg
|
||||
var tsm internalpb.TimeTickMsg
|
||||
if err := proto.Unmarshal(msg.Payload(), &tsm); err != nil {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,10 +4,11 @@ import (
|
|||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
msgPb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
)
|
||||
|
|
@ -76,7 +77,7 @@ func TestSegment_SegmentInsert(t *testing.T) {
|
|||
rawData = append(rawData, bs...)
|
||||
var records []*commonpb.Blob
|
||||
for i := 0; i < N; i++ {
|
||||
blob := &commonpb.Blob {
|
||||
blob := &commonpb.Blob{
|
||||
Value: rawData,
|
||||
}
|
||||
records = append(records, blob)
|
||||
|
|
@ -177,7 +178,7 @@ func TestSegment_SegmentSearch(t *testing.T) {
|
|||
rawData = append(rawData, bs...)
|
||||
var records []*commonpb.Blob
|
||||
for i := 0; i < N; i++ {
|
||||
blob := &commonpb.Blob {
|
||||
blob := &commonpb.Blob{
|
||||
Value: rawData,
|
||||
}
|
||||
ids = append(ids, int64(i))
|
||||
|
|
@ -319,7 +320,7 @@ func TestSegment_GetRowCount(t *testing.T) {
|
|||
rawData = append(rawData, bs...)
|
||||
var records []*commonpb.Blob
|
||||
for i := 0; i < N; i++ {
|
||||
blob := &commonpb.Blob {
|
||||
blob := &commonpb.Blob{
|
||||
Value: rawData,
|
||||
}
|
||||
records = append(records, blob)
|
||||
|
|
@ -429,7 +430,7 @@ func TestSegment_GetMemSize(t *testing.T) {
|
|||
rawData = append(rawData, bs...)
|
||||
var records []*commonpb.Blob
|
||||
for i := 0; i < N; i++ {
|
||||
blob := &commonpb.Blob {
|
||||
blob := &commonpb.Blob{
|
||||
Value: rawData,
|
||||
}
|
||||
records = append(records, blob)
|
||||
|
|
|
|||
|
|
@ -1,144 +0,0 @@
|
|||
package etcdutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pingcap/log"
|
||||
"github.com/zilliztech/milvus-distributed/internal/errors"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"go.etcd.io/etcd/etcdserver"
|
||||
"go.etcd.io/etcd/pkg/types"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultDialTimeout is the maximum amount of time a dial will wait for a
|
||||
// connection to setup. 30s is long enough for most of the network conditions.
|
||||
DefaultDialTimeout = 30 * time.Second
|
||||
|
||||
// DefaultRequestTimeout 10s is long enough for most of etcd clusters.
|
||||
DefaultRequestTimeout = 10 * time.Second
|
||||
|
||||
// DefaultSlowRequestTime 1s for the threshold for normal request, for those
|
||||
// longer then 1s, they are considered as slow requests.
|
||||
DefaultSlowRequestTime = 1 * time.Second
|
||||
)
|
||||
|
||||
// CheckClusterID checks Etcd's cluster ID, returns an error if mismatch.
|
||||
// This function will never block even quorum is not satisfied.
|
||||
func CheckClusterID(localClusterID types.ID, um types.URLsMap, tlsConfig *tls.Config) error {
|
||||
if len(um) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var peerURLs []string
|
||||
for _, urls := range um {
|
||||
peerURLs = append(peerURLs, urls.StringSlice()...)
|
||||
}
|
||||
|
||||
for _, u := range peerURLs {
|
||||
trp := &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
}
|
||||
remoteCluster, gerr := etcdserver.GetClusterFromRemotePeers(nil, []string{u}, trp)
|
||||
trp.CloseIdleConnections()
|
||||
if gerr != nil {
|
||||
// Do not return error, because other members may be not ready.
|
||||
log.Error("failed to get cluster from remote")
|
||||
continue
|
||||
}
|
||||
|
||||
remoteClusterID := remoteCluster.ID()
|
||||
if remoteClusterID != localClusterID {
|
||||
return errors.Errorf("Etcd cluster ID mismatch, expect %d, got %d", localClusterID, remoteClusterID)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddEtcdMember adds an etcd member.
|
||||
func AddEtcdMember(client *clientv3.Client, urls []string) (*clientv3.MemberAddResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(client.Ctx(), DefaultRequestTimeout)
|
||||
addResp, err := client.MemberAdd(ctx, urls)
|
||||
cancel()
|
||||
return addResp, errors.WithStack(err)
|
||||
}
|
||||
|
||||
// ListEtcdMembers returns a list of internal etcd members.
|
||||
func ListEtcdMembers(client *clientv3.Client) (*clientv3.MemberListResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(client.Ctx(), DefaultRequestTimeout)
|
||||
listResp, err := client.MemberList(ctx)
|
||||
cancel()
|
||||
return listResp, errors.WithStack(err)
|
||||
}
|
||||
|
||||
// RemoveEtcdMember removes a member by the given id.
|
||||
func RemoveEtcdMember(client *clientv3.Client, id uint64) (*clientv3.MemberRemoveResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(client.Ctx(), DefaultRequestTimeout)
|
||||
rmResp, err := client.MemberRemove(ctx, id)
|
||||
cancel()
|
||||
return rmResp, errors.WithStack(err)
|
||||
}
|
||||
|
||||
// EtcdKVGet returns the etcd GetResponse by given key or key prefix
|
||||
func EtcdKVGet(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
|
||||
ctx, cancel := context.WithTimeout(c.Ctx(), DefaultRequestTimeout)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
resp, err := clientv3.NewKV(c).Get(ctx, key, opts...)
|
||||
if err != nil {
|
||||
log.Error("load from etcd meet error")
|
||||
}
|
||||
if cost := time.Since(start); cost > DefaultSlowRequestTime {
|
||||
log.Warn("kv gets too slow", zap.String("request-key", key), zap.Duration("cost", cost), zap.Error(err))
|
||||
}
|
||||
|
||||
return resp, errors.WithStack(err)
|
||||
}
|
||||
|
||||
// GetValue gets value with key from etcd.
|
||||
func GetValue(c *clientv3.Client, key string, opts ...clientv3.OpOption) ([]byte, error) {
|
||||
resp, err := get(c, key, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return resp.Kvs[0].Value, nil
|
||||
}
|
||||
|
||||
func get(c *clientv3.Client, key string, opts ...clientv3.OpOption) (*clientv3.GetResponse, error) {
|
||||
resp, err := EtcdKVGet(c, key, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if n := len(resp.Kvs); n == 0 {
|
||||
return nil, nil
|
||||
} else if n > 1 {
|
||||
return nil, errors.Errorf("invalid get value resp %v, must only one", resp.Kvs)
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// GetProtoMsgWithModRev returns boolean to indicate whether the key exists or not.
|
||||
func GetProtoMsgWithModRev(c *clientv3.Client, key string, msg proto.Message, opts ...clientv3.OpOption) (bool, int64, error) {
|
||||
resp, err := get(c, key, opts...)
|
||||
if err != nil {
|
||||
return false, 0, err
|
||||
}
|
||||
if resp == nil {
|
||||
return false, 0, nil
|
||||
}
|
||||
value := resp.Kvs[0].Value
|
||||
if err = proto.Unmarshal(value, msg); err != nil {
|
||||
return false, 0, errors.WithStack(err)
|
||||
}
|
||||
return true, resp.Kvs[0].ModRevision, nil
|
||||
}
|
||||
|
|
@ -1,176 +0,0 @@
|
|||
package etcdutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
. "github.com/pingcap/check"
|
||||
"github.com/tikv/pd/pkg/tempurl"
|
||||
"go.etcd.io/etcd/clientv3"
|
||||
"go.etcd.io/etcd/embed"
|
||||
"go.etcd.io/etcd/pkg/types"
|
||||
)
|
||||
|
||||
func Test(t *testing.T) {
|
||||
TestingT(t)
|
||||
}
|
||||
|
||||
var _ = Suite(&testEtcdutilSuite{})
|
||||
|
||||
type testEtcdutilSuite struct{}
|
||||
|
||||
func newTestSingleConfig() *embed.Config {
|
||||
cfg := embed.NewConfig()
|
||||
cfg.Name = "test_etcd"
|
||||
cfg.Dir, _ = ioutil.TempDir("/tmp", "test_etcd")
|
||||
cfg.WalDir = ""
|
||||
cfg.Logger = "zap"
|
||||
cfg.LogOutputs = []string{"stdout"}
|
||||
|
||||
pu, _ := url.Parse(tempurl.Alloc())
|
||||
cfg.LPUrls = []url.URL{*pu}
|
||||
cfg.APUrls = cfg.LPUrls
|
||||
cu, _ := url.Parse(tempurl.Alloc())
|
||||
cfg.LCUrls = []url.URL{*cu}
|
||||
cfg.ACUrls = cfg.LCUrls
|
||||
|
||||
cfg.StrictReconfigCheck = false
|
||||
cfg.InitialCluster = fmt.Sprintf("%s=%s", cfg.Name, &cfg.LPUrls[0])
|
||||
cfg.ClusterState = embed.ClusterStateFlagNew
|
||||
return cfg
|
||||
}
|
||||
|
||||
func cleanConfig(cfg *embed.Config) {
|
||||
// Clean data directory
|
||||
os.RemoveAll(cfg.Dir)
|
||||
}
|
||||
|
||||
func (s *testEtcdutilSuite) TestMemberHelpers(c *C) {
|
||||
cfg1 := newTestSingleConfig()
|
||||
etcd1, err := embed.StartEtcd(cfg1)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
ep1 := cfg1.LCUrls[0].String()
|
||||
client1, err := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{ep1},
|
||||
})
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
<-etcd1.Server.ReadyNotify()
|
||||
|
||||
// Test ListEtcdMembers
|
||||
listResp1, err := ListEtcdMembers(client1)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(listResp1.Members), Equals, 1)
|
||||
// types.ID is an alias of uint64.
|
||||
c.Assert(listResp1.Members[0].ID, Equals, uint64(etcd1.Server.ID()))
|
||||
|
||||
// Test AddEtcdMember
|
||||
// Make a new etcd config.
|
||||
cfg2 := newTestSingleConfig()
|
||||
cfg2.Name = "etcd2"
|
||||
cfg2.InitialCluster = cfg1.InitialCluster + fmt.Sprintf(",%s=%s", cfg2.Name, &cfg2.LPUrls[0])
|
||||
cfg2.ClusterState = embed.ClusterStateFlagExisting
|
||||
|
||||
// Add it to the cluster above.
|
||||
peerURL := cfg2.LPUrls[0].String()
|
||||
addResp, err := AddEtcdMember(client1, []string{peerURL})
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
etcd2, err := embed.StartEtcd(cfg2)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(addResp.Member.ID, Equals, uint64(etcd2.Server.ID()))
|
||||
|
||||
ep2 := cfg2.LCUrls[0].String()
|
||||
client2, err := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{ep2},
|
||||
})
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
<-etcd2.Server.ReadyNotify()
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
listResp2, err := ListEtcdMembers(client2)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(listResp2.Members), Equals, 2)
|
||||
for _, m := range listResp2.Members {
|
||||
switch m.ID {
|
||||
case uint64(etcd1.Server.ID()):
|
||||
case uint64(etcd2.Server.ID()):
|
||||
default:
|
||||
c.Fatalf("unknown member: %v", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Test CheckClusterID
|
||||
urlmap, err := types.NewURLsMap(cfg2.InitialCluster)
|
||||
c.Assert(err, IsNil)
|
||||
err = CheckClusterID(etcd1.Server.Cluster().ID(), urlmap, &tls.Config{})
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
// Test RemoveEtcdMember
|
||||
_, err = RemoveEtcdMember(client1, uint64(etcd2.Server.ID()))
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
listResp3, err := ListEtcdMembers(client1)
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(listResp3.Members), Equals, 1)
|
||||
c.Assert(listResp3.Members[0].ID, Equals, uint64(etcd1.Server.ID()))
|
||||
|
||||
etcd1.Close()
|
||||
etcd2.Close()
|
||||
cleanConfig(cfg1)
|
||||
cleanConfig(cfg2)
|
||||
}
|
||||
|
||||
func (s *testEtcdutilSuite) TestEtcdKVGet(c *C) {
|
||||
cfg := newTestSingleConfig()
|
||||
etcd, err := embed.StartEtcd(cfg)
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
ep := cfg.LCUrls[0].String()
|
||||
client, err := clientv3.New(clientv3.Config{
|
||||
Endpoints: []string{ep},
|
||||
})
|
||||
c.Assert(err, IsNil)
|
||||
|
||||
<-etcd.Server.ReadyNotify()
|
||||
|
||||
keys := []string{"test/key1", "test/key2", "test/key3", "test/key4", "test/key5"}
|
||||
vals := []string{"val1", "val2", "val3", "val4", "val5"}
|
||||
|
||||
kv := clientv3.NewKV(client)
|
||||
for i := range keys {
|
||||
_, err = kv.Put(context.TODO(), keys[i], vals[i])
|
||||
c.Assert(err, IsNil)
|
||||
}
|
||||
|
||||
// Test simple point get
|
||||
resp, err := EtcdKVGet(client, "test/key1")
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(string(resp.Kvs[0].Value), Equals, "val1")
|
||||
|
||||
// Test range get
|
||||
withRange := clientv3.WithRange("test/zzzz")
|
||||
withLimit := clientv3.WithLimit(3)
|
||||
resp, err = EtcdKVGet(client, "test/", withRange, withLimit, clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(resp.Kvs), Equals, 3)
|
||||
|
||||
for i := range resp.Kvs {
|
||||
c.Assert(string(resp.Kvs[i].Key), Equals, keys[i])
|
||||
c.Assert(string(resp.Kvs[i].Value), Equals, vals[i])
|
||||
}
|
||||
|
||||
lastKey := string(resp.Kvs[len(resp.Kvs)-1].Key)
|
||||
next := clientv3.GetPrefixRangeEnd(lastKey)
|
||||
resp, err = EtcdKVGet(client, next, withRange, withLimit, clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend))
|
||||
c.Assert(err, IsNil)
|
||||
c.Assert(len(resp.Kvs), Equals, 2)
|
||||
cleanConfig(cfg)
|
||||
}
|
||||
|
|
@ -1,60 +1,57 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
//"context"
|
||||
//"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
msgpb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
"github.com/zilliztech/milvus-distributed/internal/writer"
|
||||
)
|
||||
|
||||
type Timestamp = typeutil.Timestamp
|
||||
|
||||
func GetInsertMsg(collectionName string, partitionTag string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
|
||||
return &msgpb.InsertOrDeleteMsg{
|
||||
CollectionName: collectionName,
|
||||
PartitionTag: partitionTag,
|
||||
SegmentId: UniqueID(entityId / 100),
|
||||
Uid: UniqueID(entityId),
|
||||
Timestamp: Timestamp(entityId),
|
||||
ClientId: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func GetDeleteMsg(collectionName string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
|
||||
return &msgpb.InsertOrDeleteMsg{
|
||||
CollectionName: collectionName,
|
||||
Uid: entityId,
|
||||
Timestamp: Timestamp(entityId + 100),
|
||||
}
|
||||
}
|
||||
//func GetInsertMsg(collectionName string, partitionTag string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
|
||||
// return &msgpb.InsertOrDeleteMsg{
|
||||
// CollectionName: collectionName,
|
||||
// PartitionTag: partitionTag,
|
||||
// SegmentId: UniqueID(entityId / 100),
|
||||
// Uid: UniqueID(entityId),
|
||||
// Timestamp: Timestamp(entityId),
|
||||
// ClientId: 0,
|
||||
// }
|
||||
//}
|
||||
//
|
||||
//func GetDeleteMsg(collectionName string, entityId UniqueID) *msgpb.InsertOrDeleteMsg {
|
||||
// return &msgpb.InsertOrDeleteMsg{
|
||||
// CollectionName: collectionName,
|
||||
// Uid: entityId,
|
||||
// Timestamp: Timestamp(entityId + 100),
|
||||
// }
|
||||
//}
|
||||
|
||||
func TestInsert(t *testing.T) {
|
||||
// TODO: fix test
|
||||
return
|
||||
ctx := context.Background()
|
||||
var topics []string
|
||||
topics = append(topics, "test")
|
||||
topics = append(topics, "test1")
|
||||
writerNode, _ := writer.NewWriteNode(ctx, "null", topics, 0)
|
||||
var insertMsgs []*msgpb.InsertOrDeleteMsg
|
||||
for i := 0; i < 120; i++ {
|
||||
insertMsgs = append(insertMsgs, GetInsertMsg("collection0", "tag01", UniqueID(i)))
|
||||
}
|
||||
wg := sync.WaitGroup{}
|
||||
wg.Add(3)
|
||||
//var wg sync.WaitGroup
|
||||
writerNode.InsertBatchData(ctx, insertMsgs, &wg)
|
||||
var insertMsgs2 []*msgpb.InsertOrDeleteMsg
|
||||
for i := 120; i < 200; i++ {
|
||||
insertMsgs2 = append(insertMsgs2, GetInsertMsg("collection0", "tag02", UniqueID(i)))
|
||||
}
|
||||
writerNode.InsertBatchData(ctx, insertMsgs2, &wg)
|
||||
var deleteMsgs []*msgpb.InsertOrDeleteMsg
|
||||
deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 2))
|
||||
deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 120))
|
||||
writerNode.DeleteBatchData(ctx, deleteMsgs)
|
||||
//ctx := context.Background()
|
||||
//var topics []string
|
||||
//topics = append(topics, "test")
|
||||
//topics = append(topics, "test1")
|
||||
//writerNode, _ := writer.NewWriteNode(ctx, "null", topics, 0)
|
||||
//var insertMsgs []*msgpb.InsertOrDeleteMsg
|
||||
//for i := 0; i < 120; i++ {
|
||||
// insertMsgs = append(insertMsgs, GetInsertMsg("collection0", "tag01", UniqueID(i)))
|
||||
//}
|
||||
//wg := sync.WaitGroup{}
|
||||
//wg.Add(3)
|
||||
////var wg sync.WaitGroup
|
||||
//writerNode.InsertBatchData(ctx, insertMsgs, &wg)
|
||||
//var insertMsgs2 []*msgpb.InsertOrDeleteMsg
|
||||
//for i := 120; i < 200; i++ {
|
||||
// insertMsgs2 = append(insertMsgs2, GetInsertMsg("collection0", "tag02", UniqueID(i)))
|
||||
//}
|
||||
//writerNode.InsertBatchData(ctx, insertMsgs2, &wg)
|
||||
//var deleteMsgs []*msgpb.InsertOrDeleteMsg
|
||||
//deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 2))
|
||||
//deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 120))
|
||||
//writerNode.DeleteBatchData(ctx, deleteMsgs)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,9 @@
|
|||
package test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"testing"
|
||||
|
||||
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
|
||||
|
||||
"github.com/apache/pulsar-client-go/pulsar"
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/stretchr/testify/assert"
|
||||
msgpb "github.com/zilliztech/milvus-distributed/internal/proto/message"
|
||||
)
|
||||
|
||||
type UniqueID = typeutil.UniqueID
|
||||
|
|
@ -18,29 +11,29 @@ type UniqueID = typeutil.UniqueID
|
|||
func TestKey2Seg(t *testing.T) {
|
||||
// TODO: fix test
|
||||
return
|
||||
|
||||
lookupUrl := "pulsar://localhost:6650"
|
||||
client, err := pulsar.NewClient(pulsar.ClientOptions{
|
||||
URL: lookupUrl,
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
consumer, err := client.Subscribe(pulsar.ConsumerOptions{
|
||||
Topic: "Key2Seg",
|
||||
SubscriptionName: "sub-1",
|
||||
})
|
||||
|
||||
obj := msgpb.Key2SegMsg{}
|
||||
msg, err := consumer.Receive(context.Background())
|
||||
proto.Unmarshal(msg.Payload(), &obj)
|
||||
assert.Equal(t, obj.Uid, UniqueID(0))
|
||||
consumer.Ack(msg)
|
||||
msg, err = consumer.Receive(context.Background())
|
||||
proto.Unmarshal(msg.Payload(), &obj)
|
||||
assert.Equal(t, obj.Uid, UniqueID(0))
|
||||
consumer.Ack(msg)
|
||||
consumer.Close()
|
||||
client.Close()
|
||||
//
|
||||
//lookupUrl := "pulsar://localhost:6650"
|
||||
//client, err := pulsar.NewClient(pulsar.ClientOptions{
|
||||
// URL: lookupUrl,
|
||||
//})
|
||||
//if err != nil {
|
||||
// log.Fatal(err)
|
||||
//}
|
||||
//
|
||||
//consumer, err := client.Subscribe(pulsar.ConsumerOptions{
|
||||
// Topic: "Key2Seg",
|
||||
// SubscriptionName: "sub-1",
|
||||
//})
|
||||
//
|
||||
//obj := msgpb.Key2SegMsg{}
|
||||
//msg, err := consumer.Receive(context.Background())
|
||||
//proto.Unmarshal(msg.Payload(), &obj)
|
||||
//assert.Equal(t, obj.Uid, UniqueID(0))
|
||||
//consumer.Ack(msg)
|
||||
//msg, err = consumer.Receive(context.Background())
|
||||
//proto.Unmarshal(msg.Payload(), &obj)
|
||||
//assert.Equal(t, obj.Uid, UniqueID(0))
|
||||
//consumer.Ack(msg)
|
||||
//consumer.Close()
|
||||
//client.Close()
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue