milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix bug and enchance system 

Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>
pull/4973/head^2
zhenshan.cao 2021-03-22 16:36:10 +08:00 committed by yefu.chen
parent 0341a38ee5
commit c2734fa55f
76 changed files with 1784 additions and 1170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.jenkins/modules/Regression/PythonRegression.groovy
View File

@ -54,7 +54,7 @@ timeout(time: 150, unit: 'MINUTES') {
echo "This is Cron Job!"
sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
} else {
sh "pytest --tags=0331+l1 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
sh "pytest --tags=0331+l1 -n 2 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
}
}
} catch (exc) {

5
cmd/singlenode/main.go
View File

@ -29,7 +29,12 @@ func initLogCfg() log.Config {
logCfg.File.MaxSize = 300
logCfg.File.MaxBackups = 20
logCfg.File.MaxDays = 10
ciFileDir := "/milvus-distributed/logs/"
if _, err := os.Stat(ciFileDir); err == nil {
logCfg.File.Filename = ciFileDir + "singlenode.log"
} else {
logCfg.File.Filename = "/tmp/milvus/singlenode.log"
}
return logCfg
}

4
go.mod
View File

@ -4,7 +4,7 @@ go 1.15
require (
github.com/HdrHistogram/hdrhistogram-go v1.0.1 // indirect
github.com/apache/pulsar-client-go v0.1.1
github.com/apache/pulsar-client-go v0.3.0
github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
@ -14,7 +14,7 @@ require (
github.com/gogo/protobuf v1.3.1
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/mock v1.3.1
github.com/golang/protobuf v1.3.2
github.com/golang/protobuf v1.4.2
github.com/google/btree v1.0.0
github.com/jarcoal/httpmock v1.0.8
github.com/klauspost/compress v1.10.11 // indirect

66
go.sum
View File

@ -10,6 +10,8 @@ cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqCl
cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
github.com/99designs/keyring v1.1.5 h1:wLv7QyzYpFIyMSwOADq1CLTF9KbjbBfcnfmOGJ64aO4=
github.com/99designs/keyring v1.1.5/go.mod h1:7hsVvt2qXgtadGevGJ4ujg+u8m6SpJ5TpHqTozIPqf0=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@ -17,9 +19,15 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bE
github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/apache/pulsar-client-go v0.1.1 h1:v/kU+2ZCC6yFIcbZrFtWa9/nvVzVr18L+xYJUvZSxEQ=
github.com/apache/pulsar-client-go v0.1.1/go.mod h1:mlxC65KL1BLhGO2bnT9zWMttVzR2czVPb27D477YpyU=
github.com/apache/pulsar-client-go v0.3.0 h1:rNhJ/ENwoEfZPHHwUHNxPBTNqNQE2LQEm7DXu043giM=
github.com/apache/pulsar-client-go v0.3.0/go.mod h1:9eSgOadVhCfb2DfWtS1SCYaYIMk9VDOZztr4u3FO8cQ=
github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb h1:E1P0FudxDdj2RhbveZC9i3PwukLCA/4XQSkBS/dw6/I=
github.com/apache/pulsar-client-go/oauth2 v0.0.0-20200715083626-b9f8c5cedefb/go.mod h1:0UtvvETGDdvXNDCHa8ZQpxl+w3HbdFtfYZvDHLgWGTY=
github.com/apache/thrift v0.14.1 h1:Yh8v0hpCj63p5edXOLaqTJW0IJ1p+eMW6+YSOqw1d6s=
github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4 h1:orNYqmQGnSjgOauLWjHEp9/qIDT98xv/0Aa4Zet3/Y8=
github.com/apache/thrift/lib/go/thrift v0.0.0-20210120171102-e27e82c46ba4/go.mod h1:V/LzksIyqd3KZuQ2SunvReTG/UkArhII1dAWY5U1sCE=
@ -35,6 +43,8 @@ github.com/beefsack/go-rate v0.0.0-20180408011153-efa7637bb9b6/go.mod h1:6YNgTHL
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
@ -42,6 +52,8 @@ github.com/boynton/repl v0.0.0-20170116235056-348863958e3e/go.mod h1:Crc/GCZ3NXD
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y=
github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
github.com/coreos/bbolt v1.3.2 h1:wZwiHHUieZCquLkDL0B8UhzreNWsPHooDAG3q34zk0s=
@ -58,6 +70,10 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbp
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/danieljoos/wincred v1.0.2 h1:zf4bhty2iLuwgjgpraD2E9UbvO+fe54XXGJbOwe23fU=
github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3EJbmjhLdK9U=
github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32 h1:QWqadCIHYA5zja4b6h9uGQn93u1vL+G/aewImumdg/M=
github.com/datadog/zstd v1.4.6-0.20200617134701-89f69fb7df32/go.mod h1:inRp+etsHuvVqMPNTXaFlpf/Tj7wqviBtdJoPVrPEFQ=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -68,6 +84,8 @@ github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4w
github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a h1:mq+R6XEM6lJX5VlLyZIrUSP8tSuJp82xTK89hvBwJbU=
github.com/dvsekhvalnov/jose2go v0.0.0-20180829124132-7f401d37b68a/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
@ -81,17 +99,22 @@ github.com/frankban/quicktest v1.10.2 h1:19ARM85nVi4xH7xPXuc5eM/udya5ieh7b/Sv+d8
github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s=
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-basic/ipv4 v1.0.0 h1:gjyFAa1USC1hhXTkPOwBWDPfMcUaIM+tvo1XzV9EZxs=
github.com/go-basic/ipv4 v1.0.0/go.mod h1:etLBnaxbidQfuqE6wgZQfs38nEWNmzALkxDZe4xY8Dg=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0=
github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@ -119,6 +142,8 @@ github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@ -155,6 +180,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI=
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@ -176,6 +203,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jarcoal/httpmock v1.0.8 h1:8kI16SoO6LQKgPE7PvQuV+YuD/inwHd7fOOe2zMbo4k=
github.com/jarcoal/httpmock v1.0.8/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
@ -194,6 +222,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d h1:Z+RDyXzjKE0i2sTjZ/b1uxiGtPhFy34Ou/Tk0qwN0kM=
github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@ -216,6 +246,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/linkedin/goavro/v2 v2.9.8 h1:jN50elxBsGBDGVDEKqUlDuU1cFwJ11K/yrJCBMe/7Wg=
github.com/linkedin/goavro/v2 v2.9.8/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@ -247,11 +279,19 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs=
github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486 h1:K35HCWaOTJIPW6cDHK4yj3QfRY/NhE0pBbfoc0M2NMQ=
github.com/opentracing-contrib/go-grpc v0.0.0-20200813121455-4a6760c71486/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@ -276,18 +316,26 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/protocolbuffers/protobuf v3.15.3+incompatible h1:5WExaSYHEGvU73sVHvqe+3/APOOyCVg/pDCeAlfpCrw=
github.com/protocolbuffers/protobuf v3.15.4+incompatible h1:Blv4dGFGqHXX+r5Tqoc1ziXPMDElqZ+/ryYcE4bddN4=
@ -301,6 +349,7 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@ -406,6 +455,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -417,11 +467,13 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190921015927-1a5e07d1ff72/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
@ -429,13 +481,18 @@ golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwY
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -447,10 +504,17 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190804053845-51ab0e2deafa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -535,6 +599,7 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
@ -542,6 +607,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXL
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

8
internal/core/src/query/SearchOnSealed.cpp
View File

@ -29,8 +29,15 @@ AssembleNegBitset(const BitsetSimple& bitset_simple) {
for (auto& bitset : bitset_simple) {
N += bitset.size();
}
aligned_vector<uint8_t> result(upper_align(upper_div(N, 8), 64));
if (bitset_simple.size() == 1) {
auto& bitset = bitset_simple[0];
auto byte_count = upper_div(bitset.size(), 8);
auto src_ptr = boost_ext::get_data(bitset);
memcpy(result.data(), src_ptr, byte_count);
} else {
auto acc_byte_count = 0;
for (auto& bitset : bitset_simple) {
auto size = bitset.size();
@ -40,6 +47,7 @@ AssembleNegBitset(const BitsetSimple& bitset_simple) {
memcpy(result.data() + acc_byte_count, src_ptr, byte_count);
acc_byte_count += byte_count;
}
}
// revert the bitset
for (int64_t i = 0; i < result.size(); ++i) {

9
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -206,7 +206,14 @@ ExecExprVisitor::ExecRangeVisitorDispatcher(RangeExpr& expr_raw) -> RetType {
T val1, val2;
std::tie(op1, val1) = conditions[0];
std::tie(op2, val2) = conditions[1];
Assert(val1 <= val2);
// TODO: disable check?
if (val1 > val2) {
// Empty
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
RetType ret(num_chunk, boost::dynamic_bitset<>(size_per_chunk));
return ret;
}
auto ops = std::make_tuple(op1, op2);
if (false) {
} else if (ops == std::make_tuple(OpType::GreaterThan, OpType::LessThan)) {

37
internal/core/src/segcore/segment_c.cpp
View File

@ -260,6 +260,43 @@ UpdateSealedSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_inde
}
}
CStatus
DropFieldData(CSegmentInterface c_segment, int64_t field_id) {
try {
auto segment_interface = reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto segment = dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
AssertInfo(segment != nullptr, "segment conversion failed");
segment->DropFieldData(milvus::FieldId(field_id));
auto status = CStatus();
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
auto status = CStatus();
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
return status;
}
}
CStatus
DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id) {
auto status = CStatus();
try {
auto segment_interface = reinterpret_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto segment = dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
AssertInfo(segment != nullptr, "segment conversion failed");
segment->DropIndex(milvus::FieldId(field_id));
status.error_code = Success;
status.error_msg = "";
return status;
} catch (std::exception& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
return status;
}
}
////////////////////////////// deprecated interfaces //////////////////////////////
CStatus
UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info) {

8
internal/core/src/segcore/segment_c.h
View File

@ -80,13 +80,19 @@ Delete(CSegmentInterface c_segment,
int64_t
PreDelete(CSegmentInterface c_segment, int64_t size);
////////////////////////////// interfaces for growing segment //////////////////////////////
////////////////////////////// interfaces for sealed segment //////////////////////////////
CStatus
LoadFieldData(CSegmentInterface c_segment, CLoadFieldDataInfo load_field_data_info);
CStatus
UpdateSealedSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info);
CStatus
DropFieldData(CSegmentInterface c_segment, int64_t field_id);
CStatus
DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
////////////////////////////// deprecated interfaces //////////////////////////////
CStatus
UpdateSegmentIndex(CSegmentInterface c_segment, CLoadIndexInfo c_load_index_info);

51
internal/core/unittest/test_query.cpp
View File

@ -223,6 +223,57 @@ TEST(Query, ExecWithPredicateLoader) {
ASSERT_EQ(json.dump(2), ref.dump(2));
}
TEST(Query, ExecWithPredicateSmallN) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 7, MetricType::METRIC_L2);
schema->AddDebugField("age", DataType::FLOAT);
std::string dsl = R"({
"bool": {
"must": [
{
"range": {
"age": {
"GE": -1,
"LT": 1
}
}
},
{
"vector": {
"fakevec": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 5
}
}
}
]
}
})";
int64_t N = 177;
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema);
segment->PreInsert(N);
segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
auto plan = CreatePlan(*schema, dsl);
auto num_queries = 5;
auto ph_group_raw = CreatePlaceholderGroup(num_queries, 7, 1024);
auto ph_group = ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
Timestamp time = 1000000;
std::vector<const PlaceholderGroup*> ph_group_arr = {ph_group.get()};
auto qr = segment->Search(plan.get(), ph_group_arr.data(), &time, 1);
int topk = 5;
Json json = QueryResultToJson(qr);
std::cout << json.dump(2);
}
TEST(Query, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;

2
internal/datanode/collection_replica.go
View File

@ -46,6 +46,7 @@ type Segment struct {
endTime Timestamp // not using
startPosition *internalpb.MsgPosition
endPosition *internalpb.MsgPosition // not using
channelName string
}
type CollectionSegmentReplica struct {
@ -99,6 +100,7 @@ func (replica *CollectionSegmentReplica) addSegment(
createTime: 0,
startPosition: position,
endPosition: new(internalpb.MsgPosition),
channelName: channelName,
}
seg.isNew.Store(true)

2
internal/datanode/data_sync_service.go
View File

@ -73,7 +73,7 @@ func (dsService *dataSyncService) initNodes() {
}
return nil
}
err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
panic(err)
}

31
internal/datanode/flow_graph_insert_buffer_node.go
View File

@ -117,7 +117,21 @@ func (ibNode *insertBufferNode) Operate(ctx context.Context, in []Msg) ([]Msg, c
case iMsg.startPositions == nil || len(iMsg.startPositions) <= 0:
log.Error("insert Msg StartPosition empty")
default:
ibNode.replica.setStartPosition(currentSegID, iMsg.startPositions[0])
segment, err := ibNode.replica.getSegmentByID(currentSegID)
if err != nil {
log.Error("get segment wrong", zap.Error(err))
}
var startPosition *internalpb.MsgPosition = nil
for _, pos := range iMsg.startPositions {
if pos.ChannelName == segment.channelName {
startPosition = pos
}
}
if startPosition == nil {
log.Error("get position wrong", zap.Error(err))
} else {
ibNode.replica.setStartPosition(currentSegID, startPosition)
}
}
}
@ -418,7 +432,20 @@ func (ibNode *insertBufferNode) Operate(ctx context.Context, in []Msg) ([]Msg, c
case iMsg.endPositions == nil || len(iMsg.endPositions) <= 0:
log.Error("insert Msg EndPosition empty")
default:
ibNode.replica.setEndPosition(currentSegID, iMsg.endPositions[0])
segment, err := ibNode.replica.getSegmentByID(currentSegID)
if err != nil {
log.Error("get segment wrong", zap.Error(err))
}
var endPosition *internalpb.MsgPosition = nil
for _, pos := range iMsg.endPositions {
if pos.ChannelName == segment.channelName {
endPosition = pos
}
}
if endPosition == nil {
log.Error("get position wrong", zap.Error(err))
}
ibNode.replica.setEndPosition(currentSegID, endPosition)
}
// 1.4 if full

3
internal/dataservice/meta.go
View File

@ -169,7 +169,6 @@ func (meta *meta) AddSegment(segmentInfo *datapb.SegmentInfo) error {
func (meta *meta) UpdateSegment(segmentInfo *datapb.SegmentInfo) error {
meta.ddLock.Lock()
defer meta.ddLock.Unlock()
meta.segID2Info[segmentInfo.SegmentID] = segmentInfo
if err := meta.saveSegmentInfo(segmentInfo); err != nil {
_ = meta.reloadFromKV()
@ -252,7 +251,7 @@ func (meta *meta) FlushSegment(segID UniqueID, timetick Timestamp) error {
}
segInfo.FlushedTime = timetick
segInfo.State = commonpb.SegmentState_Flushed
err := meta.saveSegmentInfo(segInfo)
if err != nil {
_ = meta.reloadFromKV()

12
internal/dataservice/server.go
View File

@ -150,7 +150,7 @@ func (s *Server) initMeta() error {
}
return nil
}
err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
return err
}
@ -343,18 +343,12 @@ func (s *Server) startSegmentFlushChannel(ctx context.Context) {
continue
}
realMsg := msg.(*msgstream.FlushCompletedMsg)
segmentInfo, err := s.meta.GetSegment(realMsg.SegmentID)
err := s.meta.FlushSegment(realMsg.SegmentID, realMsg.BeginTimestamp)
log.Debug("dataservice flushed segment", zap.Any("segmentID", realMsg.SegmentID), zap.Error(err))
if err != nil {
log.Error("get segment from meta error", zap.Int64("segmentID", realMsg.SegmentID), zap.Error(err))
continue
}
segmentInfo.FlushedTime = realMsg.BeginTimestamp
segmentInfo.State = commonpb.SegmentState_Flushed
if err = s.meta.UpdateSegment(segmentInfo); err != nil {
log.Error("update segment error", zap.Error(err))
continue
}
}
}
}

4
internal/dataservice/stats_handler.go
View File

@ -1,7 +1,9 @@
package dataservice
import (
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"go.uber.org/zap"
)
type statsHandler struct {
@ -32,6 +34,6 @@ func (handler *statsHandler) HandleSegmentStat(segStats *internalpb.SegmentStati
segMeta.SealedTime = segStats.EndTime
segMeta.NumRows = segStats.NumRows
segMeta.MemSize = segStats.MemorySize
log.Debug("stats_handler update segment", zap.Any("segmentID", segMeta.SegmentID), zap.Any("State", segMeta.State))
return handler.meta.UpdateSegment(segMeta)
}

2
internal/distributed/datanode/client/client.go
View File

@ -48,7 +48,7 @@ func (c *Client) Init() error {
return nil
}
err := retry.Retry(100, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

4
internal/distributed/datanode/service.go
View File

@ -182,7 +182,7 @@ func (s *Server) init() error {
if err = masterClient.Start(); err != nil {
panic(err)
}
err = funcutil.WaitForComponentHealthy(ctx, masterClient, "MasterService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentHealthy(ctx, masterClient, "MasterService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
@ -202,7 +202,7 @@ func (s *Server) init() error {
if err = dataService.Start(); err != nil {
panic(err)
}
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}

2
internal/distributed/dataservice/client/client.go
View File

@ -47,7 +47,7 @@ func (c *Client) Init() error {
return nil
}
err := retry.Retry(100, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

2
internal/distributed/dataservice/service.go
View File

@ -104,7 +104,7 @@ func (s *Server) init() error {
s.dataService.UpdateStateCode(internalpb.StateCode_Initializing)
ctx := context.Background()
err = funcutil.WaitForComponentInitOrHealthy(ctx, client, "MasterService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, client, "MasterService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)

2
internal/distributed/indexnode/client/client.go
View File

@ -45,7 +45,7 @@ func (c *Client) Init() error {
c.grpcClient = indexpb.NewIndexNodeClient(conn)
return nil
}
err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

29
internal/distributed/indexnode/paramtable.go
View File

@ -1,9 +1,6 @@
package grpcindexnode
import (
"net"
"os"
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/util/funcutil"
@ -35,11 +32,6 @@ func (pt *ParamTable) LoadFromArgs() {
}
func (pt *ParamTable) LoadFromEnv() {
indexServiceAddress := os.Getenv("INDEX_SERVICE_ADDRESS")
if indexServiceAddress != "" {
pt.IndexServerAddress = indexServiceAddress
}
Params.IP = funcutil.GetLocalIP()
}
@ -50,28 +42,11 @@ func (pt *ParamTable) initParams() {
// todo remove and use load from env
func (pt *ParamTable) initIndexServerAddress() {
addr, err := pt.Load("indexService.address")
ret, err := pt.Load("IndexServiceAddress")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip indexServer.address")
}
}
port, err := pt.Load("indexService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.IndexServerAddress = addr + ":" + port
pt.IndexServerAddress = ret
}
func (pt *ParamTable) initPort() {

2
internal/distributed/indexservice/client/client.go
View File

@ -50,7 +50,7 @@ func (c *Client) Init() error {
c.grpcClient = indexpb.NewIndexServiceClient(conn)
return nil
}
err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

22
internal/distributed/indexservice/paramtable.go
View File

@ -1,8 +1,6 @@
package grpcindexservice
import (
"net"
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
@ -35,25 +33,9 @@ func (pt *ParamTable) initServicePort() {
}
func (pt *ParamTable) initServiceAddress() {
addr, err := pt.Load("indexService.address")
ret, err := pt.Load("IndexServiceAddress")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip proxyService.address")
}
}
port, err := pt.Load("indexService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.ServiceAddress = addr + ":" + port
pt.ServiceAddress = ret
}

2
internal/distributed/masterservice/client/client.go
View File

@ -32,7 +32,7 @@ func NewClient(addr string, timeout time.Duration) (*GrpcClient, error) {
addr: addr,
timeout: timeout,
grpcTimeout: time.Second * 5,
retry: 3,
retry: 300,
}, nil
}

4
internal/distributed/masterservice/masterservice_test.go
View File

@ -357,6 +357,7 @@ func TestGrpcService(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, partMeta.PartitionName, "testPartition")
assert.Equal(t, 1, len(collectionMetaCache))
})
t.Run("has partition", func(t *testing.T) {
@ -600,6 +601,7 @@ func TestGrpcService(t *testing.T) {
partMeta, err := core.MetaTable.GetPartitionByID(collMeta.PartitionIDs[0])
assert.Nil(t, err)
assert.Equal(t, partMeta.PartitionName, cms.Params.DefaultPartitionName)
assert.Equal(t, 2, len(collectionMetaCache))
})
t.Run("drop collection", func(t *testing.T) {
@ -620,7 +622,7 @@ func TestGrpcService(t *testing.T) {
assert.Equal(t, dropCollectionArray[0].Base.MsgType, commonpb.MsgType_DropCollection)
assert.Equal(t, status.ErrorCode, commonpb.ErrorCode_Success)
assert.Equal(t, dropCollectionArray[0].CollectionName, "testColl")
assert.Equal(t, len(collectionMetaCache), 1)
assert.Equal(t, len(collectionMetaCache), 3)
assert.Equal(t, collectionMetaCache[0], "testColl")
req = &milvuspb.DropCollectionRequest{

4
internal/distributed/masterservice/server.go
View File

@ -123,7 +123,7 @@ func (s *Server) init() error {
panic(err)
}
err := funcutil.WaitForComponentInitOrHealthy(ctx, proxyService, "ProxyService", 100, 200*time.Millisecond)
err := funcutil.WaitForComponentInitOrHealthy(ctx, proxyService, "ProxyService", 1000000, 200*time.Millisecond)
if err != nil {
panic(err)
}
@ -141,7 +141,7 @@ func (s *Server) init() error {
if err := dataService.Start(); err != nil {
panic(err)
}
err := funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, 200*time.Millisecond)
err := funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, 200*time.Millisecond)
if err != nil {
panic(err)
}

2
internal/distributed/proxynode/client/client.go
View File

@ -44,7 +44,7 @@ func (c *Client) Init() error {
c.grpcClient = proxypb.NewProxyNodeServiceClient(conn)
return nil
}
err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

98
internal/distributed/proxynode/paramtable.go
View File

@ -1,9 +1,6 @@
package grpcproxynode
import (
"net"
"os"
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/util/funcutil"
@ -42,32 +39,6 @@ func (pt *ParamTable) LoadFromArgs() {
}
func (pt *ParamTable) LoadFromEnv() {
masterAddress := os.Getenv("MASTER_ADDRESS")
if masterAddress != "" {
pt.MasterAddress = masterAddress
}
proxyServiceAddress := os.Getenv("PROXY_SERVICE_ADDRESS")
if proxyServiceAddress != "" {
pt.ProxyServiceAddress = proxyServiceAddress
}
indexServiceAddress := os.Getenv("INDEX_SERVICE_ADDRESS")
if indexServiceAddress != "" {
pt.IndexServerAddress = indexServiceAddress
}
queryServiceAddress := os.Getenv("QUERY_SERVICE_ADDRESS")
if queryServiceAddress != "" {
pt.QueryServiceAddress = queryServiceAddress
}
dataServiceAddress := os.Getenv("DATA_SERVICE_ADDRESS")
if dataServiceAddress != "" {
pt.DataServiceAddress = dataServiceAddress
}
Params.IP = funcutil.GetLocalIP()
}
@ -86,96 +57,47 @@ func (pt *ParamTable) initPoxyServicePort() {
}
func (pt *ParamTable) initProxyServiceAddress() {
addr, err := pt.Load("proxyService.address")
ret, err := pt.Load("_PROXY_SERVICE_ADDRESS")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip proxyService.address")
}
}
port, err := pt.Load("proxyService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.ProxyServiceAddress = addr + ":" + port
pt.ProxyServiceAddress = ret
}
// todo remove and use load from env
func (pt *ParamTable) initIndexServerAddress() {
addr, err := pt.Load("indexService.address")
ret, err := pt.Load("IndexServiceAddress")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip indexService.address")
}
}
port, err := pt.Load("indexService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.IndexServerAddress = addr + ":" + port
pt.IndexServerAddress = ret
}
// todo remove and use load from env
func (pt *ParamTable) initMasterAddress() {
masterHost, err := pt.Load("master.address")
ret, err := pt.Load("_MasterAddress")
if err != nil {
panic(err)
}
port, err := pt.Load("master.port")
if err != nil {
panic(err)
}
pt.MasterAddress = masterHost + ":" + port
pt.MasterAddress = ret
}
// todo remove and use load from env
func (pt *ParamTable) initDataServiceAddress() {
addr, err := pt.Load("dataService.address")
ret, err := pt.Load("_DataServiceAddress")
if err != nil {
panic(err)
}
port, err := pt.Load("dataService.port")
if err != nil {
panic(err)
}
pt.DataServiceAddress = addr + ":" + port
pt.DataServiceAddress = ret
}
// todo remove and use load from env
func (pt *ParamTable) initQueryServiceAddress() {
addr, err := pt.Load("queryService.address")
ret, err := pt.Load("_QueryServiceAddress")
if err != nil {
panic(err)
}
port, err := pt.Load("queryService.port")
if err != nil {
panic(err)
}
pt.QueryServiceAddress = addr + ":" + port
pt.QueryServiceAddress = ret
}
func (pt *ParamTable) initPort() {

2
internal/distributed/proxynode/service.go
View File

@ -185,7 +185,7 @@ func (s *Server) init() error {
if err != nil {
return err
}
err = funcutil.WaitForComponentHealthy(ctx, s.masterServiceClient, "MasterService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentHealthy(ctx, s.masterServiceClient, "MasterService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)

2
internal/distributed/proxyservice/client/client.go
View File

@ -44,7 +44,7 @@ func (c *Client) Init() error {
c.proxyServiceClient = proxypb.NewProxyServiceClient(conn)
return nil
}
err := retry.Retry(10, time.Millisecond*200, connectGrpcFunc)
err := retry.Retry(100000, time.Millisecond*200, connectGrpcFunc)
if err != nil {
return err
}

22
internal/distributed/proxyservice/paramtable.go
View File

@ -1,8 +1,6 @@
package grpcproxyservice
import (
"net"
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/util/paramtable"
@ -35,25 +33,9 @@ func (pt *ParamTable) initServicePort() {
}
func (pt *ParamTable) initServiceAddress() {
addr, err := pt.Load("proxyService.address")
ret, err := pt.Load("_PROXY_SERVICE_ADDRESS")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip proxyService.address")
}
}
port, err := pt.Load("proxyService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.ServiceAddress = addr + ":" + port
pt.ServiceAddress = ret
}

2
internal/distributed/querynode/client/client.go
View File

@ -37,7 +37,7 @@ func (c *Client) Init() error {
ctx, cancel := context.WithTimeout(context.Background(), RPCConnectionTimeout)
defer cancel()
var err error
for i := 0; i < Retry; i++ {
for i := 0; i < Retry*100; i++ {
if c.conn, err = grpc.DialContext(ctx, c.addr, grpc.WithInsecure(), grpc.WithBlock(),
grpc.WithUnaryInterceptor(
otgrpc.OpenTracingClientInterceptor(tracer)),

8
internal/distributed/querynode/service.go
View File

@ -111,7 +111,7 @@ func (s *Server) init() error {
panic(err)
}
err = funcutil.WaitForComponentInitOrHealthy(ctx, queryService, "QueryService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, queryService, "QueryService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}
@ -139,7 +139,7 @@ func (s *Server) init() error {
panic(err)
}
err = funcutil.WaitForComponentHealthy(ctx, masterService, "MasterService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentHealthy(ctx, masterService, "MasterService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}
@ -160,7 +160,7 @@ func (s *Server) init() error {
panic(err)
}
// wait indexservice healthy
err = funcutil.WaitForComponentHealthy(ctx, indexService, "IndexService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentHealthy(ctx, indexService, "IndexService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}
@ -180,7 +180,7 @@ func (s *Server) init() error {
if err = dataService.Start(); err != nil {
panic(err)
}
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}

2
internal/distributed/queryservice/client/client.go
View File

@ -32,7 +32,7 @@ func NewClient(address string, timeout time.Duration) (*Client, error) {
conn: nil,
addr: address,
timeout: timeout,
retry: 3,
retry: 300,
}, nil
}

4
internal/distributed/queryservice/service.go
View File

@ -101,7 +101,7 @@ func (s *Server) init() error {
panic(err)
}
// wait for master init or healthy
err = funcutil.WaitForComponentInitOrHealthy(ctx, masterService, "MasterService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, masterService, "MasterService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}
@ -121,7 +121,7 @@ func (s *Server) init() error {
if err = dataService.Start(); err != nil {
panic(err)
}
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 100, time.Millisecond*200)
err = funcutil.WaitForComponentInitOrHealthy(ctx, dataService, "DataService", 1000000, time.Millisecond*200)
if err != nil {
panic(err)
}

2
internal/indexnode/indexnode.go
View File

@ -67,7 +67,7 @@ func NewIndexNode(ctx context.Context) (*IndexNode, error) {
func (i *IndexNode) Init() error {
ctx := context.Background()
err := funcutil.WaitForComponentHealthy(ctx, i.serviceClient, "IndexService", 100, time.Millisecond*200)
err := funcutil.WaitForComponentHealthy(ctx, i.serviceClient, "IndexService", 1000000, time.Millisecond*200)
if err != nil {
return err

2
internal/indexservice/indexservice.go
View File

@ -86,7 +86,7 @@ func (i *IndexService) Init() error {
i.metaTable = metakv
return nil
}
err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
return err
}

32
internal/indexservice/paramtable.go
View File

@ -1,7 +1,6 @@
package indexservice
import (
"net"
"path"
"strconv"
"sync"
@ -38,8 +37,6 @@ func (pt *ParamTable) Init() {
once.Do(func() {
pt.BaseTable.Init()
pt.initLogCfg()
pt.initAddress()
pt.initPort()
pt.initEtcdAddress()
pt.initMasterAddress()
pt.initMetaRootPath()
@ -52,35 +49,6 @@ func (pt *ParamTable) Init() {
})
}
func (pt *ParamTable) initAddress() {
addr, err := pt.Load("indexService.address")
if err != nil {
panic(err)
}
hostName, _ := net.LookupHost(addr)
if len(hostName) <= 0 {
if ip := net.ParseIP(addr); ip == nil {
panic("invalid ip indexServer.address")
}
}
port, err := pt.Load("indexService.port")
if err != nil {
panic(err)
}
_, err = strconv.Atoi(port)
if err != nil {
panic(err)
}
pt.Address = addr + ":" + port
}
func (pt *ParamTable) initPort() {
pt.Port = pt.ParseInt("indexService.port")
}
func (pt *ParamTable) initEtcdAddress() {
addr, err := pt.Load("_EtcdAddress")
if err != nil {

2
internal/kv/minio/minio_kv.go
View File

@ -43,7 +43,7 @@ func NewMinIOKV(ctx context.Context, option *Option) (*MinIOKV, error) {
return nil
}
err := retry.Retry(200, time.Millisecond*200, connectMinIOFn)
err := retry.Retry(100000, time.Millisecond*200, connectMinIOFn)
if err != nil {
return nil, err
}

4
internal/masterservice/master_service.go
View File

@ -765,7 +765,7 @@ func (c *Core) Init() error {
c.kvBase = etcdkv.NewEtcdKV(c.etcdCli, Params.KvRootPath)
return nil
}
err := retry.Retry(200, time.Millisecond*200, connectEtcdFn)
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
return
}
@ -1001,7 +1001,7 @@ func (c *Core) DescribeCollection(ctx context.Context, in *milvuspb.DescribeColl
c.ddReqQueue <- t
err := t.WaitToFinish()
if err != nil {
log.Debug("DescribeCollection Failed", zap.String("name", in.CollectionName))
log.Debug("DescribeCollection Failed", zap.String("name", in.CollectionName), zap.Error(err))
return &milvuspb.DescribeCollectionResponse{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,

14
internal/masterservice/master_service_test.go
View File

@ -508,6 +508,9 @@ func TestMasterService(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, partMsg.CollectionID, collMeta.ID)
assert.Equal(t, partMsg.PartitionID, partMeta.PartitionID)
assert.Equal(t, 1, len(pm.GetCollArray()))
assert.Equal(t, "testColl", pm.GetCollArray()[0])
})
t.Run("has partition", func(t *testing.T) {
@ -893,6 +896,9 @@ func TestMasterService(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, dmsg.CollectionID, collMeta.ID)
assert.Equal(t, dmsg.PartitionID, dropPartID)
assert.Equal(t, 2, len(pm.GetCollArray()))
assert.Equal(t, "testColl", pm.GetCollArray()[1])
})
t.Run("drop collection", func(t *testing.T) {
@ -919,8 +925,8 @@ func TestMasterService(t *testing.T) {
assert.True(t, ok)
assert.Equal(t, dmsg.CollectionID, collMeta.ID)
collArray := pm.GetCollArray()
assert.Equal(t, len(collArray), 1)
assert.Equal(t, collArray[0], "testColl")
assert.Equal(t, len(collArray), 3)
assert.Equal(t, collArray[2], "testColl")
time.Sleep(time.Millisecond * 100)
qm.mutex.Lock()
@ -944,8 +950,8 @@ func TestMasterService(t *testing.T) {
time.Sleep(time.Second)
assert.Zero(t, len(ddStream.Chan()))
collArray = pm.GetCollArray()
assert.Equal(t, len(collArray), 1)
assert.Equal(t, collArray[0], "testColl")
assert.Equal(t, len(collArray), 3)
assert.Equal(t, collArray[2], "testColl")
})
err = core.Stop()

6
internal/masterservice/task.go
View File

@ -419,6 +419,9 @@ func (t *CreatePartitionReqTask) Execute(ctx context.Context) error {
return err
}
// error doesn't matter here
_ = t.core.InvalidateCollectionMetaCache(ctx, t.Req.Base.Timestamp, t.Req.DbName, t.Req.CollectionName)
return nil
}
@ -467,6 +470,9 @@ func (t *DropPartitionReqTask) Execute(ctx context.Context) error {
if err != nil {
return err
}
// error doesn't matter here
_ = t.core.InvalidateCollectionMetaCache(ctx, t.Req.Base.Timestamp, t.Req.DbName, t.Req.CollectionName)
return nil
}

171
internal/msgstream/pulsarms/pulsar_msgstream.go
View File

@ -2,17 +2,18 @@ package pulsarms
import (
"context"
"errors"
"path/filepath"
"reflect"
"strconv"
"sync"
"time"
"errors"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/golang/protobuf/proto"
"github.com/opentracing/opentracing-go"
"go.uber.org/zap"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
@ -20,7 +21,6 @@ import (
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/util/trace"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
"go.uber.org/zap"
)
type TsMsg = msgstream.TsMsg
@ -41,8 +41,9 @@ type UnmarshalDispatcher = msgstream.UnmarshalDispatcher
type PulsarMsgStream struct {
ctx context.Context
client pulsar.Client
producers []Producer
consumers []Consumer
producers map[string]Producer
producerChannels []string
consumers map[string]Consumer
consumerChannels []string
repackFunc RepackFunc
unmarshal UnmarshalDispatcher
@ -50,6 +51,7 @@ type PulsarMsgStream struct {
wait *sync.WaitGroup
streamCancel func()
pulsarBufSize int64
producerLock *sync.Mutex
consumerLock *sync.Mutex
consumerReflects []reflect.SelectCase
@ -63,8 +65,9 @@ func newPulsarMsgStream(ctx context.Context,
unmarshal UnmarshalDispatcher) (*PulsarMsgStream, error) {
streamCtx, streamCancel := context.WithCancel(ctx)
producers := make([]Producer, 0)
consumers := make([]Consumer, 0)
producers := make(map[string]Producer)
consumers := make(map[string]Consumer)
producerChannels := make([]string, 0)
consumerChannels := make([]string, 0)
consumerReflects := make([]reflect.SelectCase, 0)
receiveBuf := make(chan *MsgPack, receiveBufSize)
@ -85,6 +88,7 @@ func newPulsarMsgStream(ctx context.Context,
ctx: streamCtx,
client: client,
producers: producers,
producerChannels: producerChannels,
consumers: consumers,
consumerChannels: consumerChannels,
unmarshal: unmarshal,
@ -92,6 +96,7 @@ func newPulsarMsgStream(ctx context.Context,
receiveBuf: receiveBuf,
streamCancel: streamCancel,
consumerReflects: consumerReflects,
producerLock: &sync.Mutex{},
consumerLock: &sync.Mutex{},
wait: &sync.WaitGroup{},
scMap: &sync.Map{},
@ -101,22 +106,24 @@ func newPulsarMsgStream(ctx context.Context,
}
func (ms *PulsarMsgStream) AsProducer(channels []string) {
for i := 0; i < len(channels); i++ {
for _, channel := range channels {
fn := func() error {
pp, err := ms.client.CreateProducer(pulsar.ProducerOptions{Topic: channels[i]})
pp, err := ms.client.CreateProducer(pulsar.ProducerOptions{Topic: channel})
if err != nil {
return err
}
if pp == nil {
return errors.New("pulsar is not ready, producer is nil")
}
ms.producers = append(ms.producers, pp)
ms.producerLock.Lock()
ms.producers[channel] = pp
ms.producerChannels = append(ms.producerChannels, channel)
ms.producerLock.Unlock()
return nil
}
err := util.Retry(20, time.Millisecond*200, fn)
if err != nil {
errMsg := "Failed to create producer " + channels[i] + ", error = " + err.Error()
errMsg := "Failed to create producer " + channel + ", error = " + err.Error()
panic(errMsg)
}
}
@ -124,14 +131,17 @@ func (ms *PulsarMsgStream) AsProducer(channels []string) {
func (ms *PulsarMsgStream) AsConsumer(channels []string,
subName string) {
for i := 0; i < len(channels); i++ {
for _, channel := range channels {
if _, ok := ms.consumers[channel]; ok {
continue
}
fn := func() error {
receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
pc, err := ms.client.Subscribe(pulsar.ConsumerOptions{
Topic: channels[i],
Topic: channel,
SubscriptionName: subName,
Type: pulsar.KeyShared,
SubscriptionInitialPosition: pulsar.SubscriptionPositionLatest,
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
MessageChannel: receiveChannel,
})
if err != nil {
@ -141,8 +151,8 @@ func (ms *PulsarMsgStream) AsConsumer(channels []string,
return errors.New("pulsar is not ready, consumer is nil")
}
ms.consumers = append(ms.consumers, pc)
ms.consumerChannels = append(ms.consumerChannels, channels[i])
ms.consumers[channel] = pc
ms.consumerChannels = append(ms.consumerChannels, channel)
ms.consumerReflects = append(ms.consumerReflects, reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(pc.Chan()),
@ -153,7 +163,7 @@ func (ms *PulsarMsgStream) AsConsumer(channels []string,
}
err := util.Retry(20, time.Millisecond*200, fn)
if err != nil {
errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg)
}
}
@ -233,6 +243,7 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
return err
}
for k, v := range result {
channel := ms.producerChannels[k]
for i := 0; i < len(v.Msgs); i++ {
mb, err := v.Msgs[i].Marshal(v.Msgs[i])
if err != nil {
@ -249,7 +260,7 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
sp, spanCtx := trace.MsgSpanFromCtx(ctx, v.Msgs[i])
trace.InjectContextToPulsarMsgProperties(sp.Context(), msg.Properties)
if _, err := ms.producers[k].Send(
if _, err := ms.producers[channel].Send(
spanCtx,
msg,
); err != nil {
@ -264,7 +275,6 @@ func (ms *PulsarMsgStream) Produce(ctx context.Context, msgPack *MsgPack) error
}
func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error {
producerLen := len(ms.producers)
for _, v := range msgPack.Msgs {
mb, err := v.Marshal(v)
if err != nil {
@ -281,8 +291,9 @@ func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) erro
sp, spanCtx := trace.MsgSpanFromCtx(ctx, v)
trace.InjectContextToPulsarMsgProperties(sp.Context(), msg.Properties)
for i := 0; i < producerLen; i++ {
if _, err := ms.producers[i].Send(
ms.producerLock.Lock()
for _, producer := range ms.producers {
if _, err := producer.Send(
spanCtx,
msg,
); err != nil {
@ -291,6 +302,7 @@ func (ms *PulsarMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) erro
return err
}
}
ms.producerLock.Unlock()
sp.Finish()
}
return nil
@ -319,7 +331,7 @@ func (ms *PulsarMsgStream) Consume() (*MsgPack, context.Context) {
sp.Finish()
return cm, ctx
case <-ms.ctx.Done():
log.Debug("context closed")
//log.Debug("context closed")
return nil, nil
}
}
@ -469,19 +481,18 @@ func (ms *PulsarMsgStream) Chan() <-chan *MsgPack {
}
func (ms *PulsarMsgStream) Seek(mp *internalpb.MsgPosition) error {
for index, channel := range ms.consumerChannels {
if channel == mp.ChannelName {
if _, ok := ms.consumers[mp.ChannelName]; ok {
consumer := ms.consumers[mp.ChannelName]
messageID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
if err != nil {
return err
}
err = ms.consumers[index].Seek(messageID)
err = consumer.Seek(messageID)
if err != nil {
return err
}
return nil
}
}
return errors.New("msgStream seek fail")
}
@ -493,6 +504,7 @@ type PulsarTtMsgStream struct {
unsolvedMutex *sync.Mutex
lastTimeStamp Timestamp
syncConsumer chan int
stopConsumeChan map[Consumer]chan bool
}
func newPulsarTtMsgStream(ctx context.Context,
@ -505,6 +517,7 @@ func newPulsarTtMsgStream(ctx context.Context,
return nil, err
}
unsolvedBuf := make(map[Consumer][]TsMsg)
stopChannel := make(map[Consumer]chan bool)
msgPositions := make(map[Consumer]*internalpb.MsgPosition)
syncConsumer := make(chan int, 1)
@ -514,19 +527,39 @@ func newPulsarTtMsgStream(ctx context.Context,
msgPositions: msgPositions,
unsolvedMutex: &sync.Mutex{},
syncConsumer: syncConsumer,
stopConsumeChan: stopChannel,
}, nil
}
func (ms *PulsarTtMsgStream) addConsumer(consumer Consumer, channel string) {
if len(ms.consumers) == 0 {
ms.syncConsumer <- 1
}
ms.consumers[channel] = consumer
ms.unsolvedBuf[consumer] = make([]TsMsg, 0)
ms.consumerChannels = append(ms.consumerChannels, channel)
ms.msgPositions[consumer] = &internalpb.MsgPosition{
ChannelName: channel,
MsgID: "",
Timestamp: ms.lastTimeStamp,
}
stopConsumeChan := make(chan bool)
ms.stopConsumeChan[consumer] = stopConsumeChan
}
func (ms *PulsarTtMsgStream) AsConsumer(channels []string,
subName string) {
for i := 0; i < len(channels); i++ {
for _, channel := range channels {
if _, ok := ms.consumers[channel]; ok {
continue
}
fn := func() error {
receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
pc, err := ms.client.Subscribe(pulsar.ConsumerOptions{
Topic: channels[i],
Topic: channel,
SubscriptionName: subName,
Type: pulsar.KeyShared,
SubscriptionInitialPosition: pulsar.SubscriptionPositionLatest,
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
MessageChannel: receiveChannel,
})
if err != nil {
@ -537,23 +570,13 @@ func (ms *PulsarTtMsgStream) AsConsumer(channels []string,
}
ms.consumerLock.Lock()
if len(ms.consumers) == 0 {
ms.syncConsumer <- 1
}
ms.consumers = append(ms.consumers, pc)
ms.unsolvedBuf[pc] = make([]TsMsg, 0)
ms.msgPositions[pc] = &internalpb.MsgPosition{
ChannelName: channels[i],
MsgID: "",
Timestamp: ms.lastTimeStamp,
}
ms.consumerChannels = append(ms.consumerChannels, channels[i])
ms.addConsumer(pc, channel)
ms.consumerLock.Unlock()
return nil
}
err := util.Retry(10, time.Millisecond*200, fn)
if err != nil {
errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg)
}
}
@ -728,41 +751,53 @@ func (ms *PulsarTtMsgStream) findTimeTick(consumer Consumer,
return
}
sp.Finish()
case <-ms.stopConsumeChan[consumer]:
return
}
}
}
func (ms *PulsarTtMsgStream) Seek(mp *internalpb.MsgPosition) error {
var consumer Consumer
var messageID MessageID
for index, channel := range ms.consumerChannels {
if filepath.Base(channel) == filepath.Base(mp.ChannelName) {
consumer = ms.consumers[index]
if len(mp.MsgID) == 0 {
// TODO:: collection should has separate channels; otherwise will consume redundant msg
messageID = pulsar.EarliestMessageID()
break
return errors.New("when msgID's length equal to 0, please use AsConsumer interface")
}
var consumer Consumer
var err error
var hasWatched bool
seekChannel := mp.ChannelName
subName := mp.MsgGroup
ms.consumerLock.Lock()
defer ms.consumerLock.Unlock()
consumer, hasWatched = ms.consumers[seekChannel]
if hasWatched {
return errors.New("the channel should has been subscribed")
}
receiveChannel := make(chan pulsar.ConsumerMessage, ms.pulsarBufSize)
consumer, err = ms.client.Subscribe(pulsar.ConsumerOptions{
Topic: seekChannel,
SubscriptionName: subName,
Type: pulsar.KeyShared,
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
MessageChannel: receiveChannel,
})
if err != nil {
return err
}
if consumer == nil {
return errors.New("pulsar is not ready, consumer is nil")
}
seekMsgID, err := typeutil.StringToPulsarMsgID(mp.MsgID)
if err != nil {
return err
}
messageID = seekMsgID
break
}
}
consumer.Seek(seekMsgID)
ms.addConsumer(consumer, seekChannel)
if consumer != nil {
err := (consumer).Seek(messageID)
if err != nil {
return err
}
if messageID == nil {
if len(consumer.Chan()) == 0 {
return nil
}
ms.unsolvedMutex.Lock()
ms.unsolvedBuf[consumer] = make([]TsMsg, 0)
for {
select {
case <-ms.ctx.Done():
@ -784,7 +819,6 @@ func (ms *PulsarTtMsgStream) Seek(mp *internalpb.MsgPosition) error {
}
if tsMsg.Type() == commonpb.MsgType_TimeTick {
if tsMsg.BeginTs() >= mp.Timestamp {
ms.unsolvedMutex.Unlock()
return nil
}
continue
@ -798,9 +832,6 @@ func (ms *PulsarTtMsgStream) Seek(mp *internalpb.MsgPosition) error {
}
}
}
}
return errors.New("msgStream seek fail")
}
func checkTimeTickMsg(msg map[Consumer]Timestamp,
@ -839,10 +870,8 @@ func checkTimeTickMsg(msg map[Consumer]Timestamp,
type InMemMsgStream struct {
buffer chan *MsgPack
}
func (ms *InMemMsgStream) Start() {}
func (ms *InMemMsgStream) Close() {}
func (ms *InMemMsgStream) ProduceOne(msg TsMsg) error {
msgPack := MsgPack{}
msgPack.BeginTs = msg.BeginTs()
@ -851,23 +880,19 @@ func (ms *InMemMsgStream) ProduceOne(msg TsMsg) error {
buffer <- &msgPack
return nil
}
func (ms *InMemMsgStream) Produce(msgPack *MsgPack) error {
buffer <- msgPack
return nil
}
func (ms *InMemMsgStream) Broadcast(msgPack *MsgPack) error {
return ms.Produce(msgPack)
}
func (ms *InMemMsgStream) Consume() *MsgPack {
select {
case msgPack := <-ms.buffer:
return msgPack
}
}
func (ms *InMemMsgStream) Chan() <- chan *MsgPack {
return buffer
}

168
internal/msgstream/rmqms/rmq_msgstream.go
View File

@ -9,14 +9,15 @@ import (
"sync"
"time"
"go.uber.org/zap"
"github.com/gogo/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/msgstream/util"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
client "github.com/zilliztech/milvus-distributed/internal/util/rocksmq/client/rocksmq"
"go.uber.org/zap"
)
type TsMsg = msgstream.TsMsg
@ -35,8 +36,9 @@ type Consumer = client.Consumer
type RmqMsgStream struct {
ctx context.Context
client client.Client
producers []Producer
consumers []Consumer
producers map[string]Producer
producerChannels []string
consumers map[string]Consumer
consumerChannels []string
unmarshal msgstream.UnmarshalDispatcher
repackFunc msgstream.RepackFunc
@ -45,6 +47,7 @@ type RmqMsgStream struct {
wait *sync.WaitGroup
streamCancel func()
rmqBufSize int64
producerLock *sync.Mutex
consumerLock *sync.Mutex
consumerReflects []reflect.SelectCase
@ -55,10 +58,11 @@ func newRmqMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int64
unmarshal msgstream.UnmarshalDispatcher) (*RmqMsgStream, error) {
streamCtx, streamCancel := context.WithCancel(ctx)
producers := make([]Producer, 0)
consumers := make([]Consumer, 0)
consumerChannels := make([]string, 0)
producers := make(map[string]Producer)
producerChannels := make([]string, 0)
consumerReflects := make([]reflect.SelectCase, 0)
consumers := make(map[string]Consumer)
consumerChannels := make([]string, 0)
receiveBuf := make(chan *MsgPack, receiveBufSize)
var clientOpts client.ClientOptions
@ -73,12 +77,14 @@ func newRmqMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int64
ctx: streamCtx,
client: client,
producers: producers,
producerChannels: producerChannels,
consumers: consumers,
consumerChannels: consumerChannels,
unmarshal: unmarshal,
receiveBuf: receiveBuf,
streamCancel: streamCancel,
consumerReflects: consumerReflects,
producerLock: &sync.Mutex{},
consumerLock: &sync.Mutex{},
wait: &sync.WaitGroup{},
scMap: &sync.Map{},
@ -92,6 +98,8 @@ func (rms *RmqMsgStream) Start() {
func (rms *RmqMsgStream) Close() {
rms.streamCancel()
rms.wait.Wait()
if rms.client != nil {
rms.client.Close()
}
@ -105,7 +113,10 @@ func (rms *RmqMsgStream) AsProducer(channels []string) {
for _, channel := range channels {
pp, err := rms.client.CreateProducer(client.ProducerOptions{Topic: channel})
if err == nil {
rms.producers = append(rms.producers, pp)
rms.producerLock.Lock()
rms.producers[channel] = pp
rms.producerChannels = append(rms.producerChannels, channel)
rms.producerLock.Unlock()
} else {
errMsg := "Failed to create producer " + channel + ", error = " + err.Error()
panic(errMsg)
@ -114,11 +125,14 @@ func (rms *RmqMsgStream) AsProducer(channels []string) {
}
func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
for i := 0; i < len(channels); i++ {
for _, channel := range channels {
if _, ok := rms.consumers[channel]; ok {
continue
}
fn := func() error {
receiveChannel := make(chan client.ConsumerMessage, rms.rmqBufSize)
pc, err := rms.client.Subscribe(client.ConsumerOptions{
Topic: channels[i],
Topic: channel,
SubscriptionName: groupName,
MessageChannel: receiveChannel,
})
@ -129,8 +143,8 @@ func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
return errors.New("RocksMQ is not ready, consumer is nil")
}
rms.consumers = append(rms.consumers, pc)
rms.consumerChannels = append(rms.consumerChannels, channels[i])
rms.consumers[channel] = pc
rms.consumerChannels = append(rms.consumerChannels, channel)
rms.consumerReflects = append(rms.consumerReflects, reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(pc.Chan()),
@ -141,7 +155,7 @@ func (rms *RmqMsgStream) AsConsumer(channels []string, groupName string) {
}
err := util.Retry(20, time.Millisecond*200, fn)
if err != nil {
errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg)
}
}
@ -194,6 +208,7 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
return err
}
for k, v := range result {
channel := rms.producerChannels[k]
for i := 0; i < len(v.Msgs); i++ {
mb, err := v.Msgs[i].Marshal(v.Msgs[i])
if err != nil {
@ -205,7 +220,7 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
return err
}
msg := &client.ProducerMessage{Payload: m}
if err := rms.producers[k].Send(msg); err != nil {
if err := rms.producers[channel].Send(msg); err != nil {
return err
}
}
@ -214,7 +229,6 @@ func (rms *RmqMsgStream) Produce(ctx context.Context, pack *msgstream.MsgPack) e
}
func (rms *RmqMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error {
producerLen := len(rms.producers)
for _, v := range msgPack.Msgs {
mb, err := v.Marshal(v)
if err != nil {
@ -228,13 +242,15 @@ func (rms *RmqMsgStream) Broadcast(ctx context.Context, msgPack *MsgPack) error
msg := &client.ProducerMessage{Payload: m}
for i := 0; i < producerLen; i++ {
if err := rms.producers[i].Send(
rms.producerLock.Lock()
for _, producer := range rms.producers {
if err := producer.Send(
msg,
); err != nil {
return err
}
}
rms.producerLock.Unlock()
}
return nil
}
@ -249,7 +265,7 @@ func (rms *RmqMsgStream) Consume() (*msgstream.MsgPack, context.Context) {
}
return cm, nil
case <-rms.ctx.Done():
log.Debug("context closed")
//log.Debug("context closed")
return nil, nil
}
}
@ -298,20 +314,18 @@ func (rms *RmqMsgStream) Chan() <-chan *msgstream.MsgPack {
}
func (rms *RmqMsgStream) Seek(mp *msgstream.MsgPosition) error {
for index, channel := range rms.consumerChannels {
if channel == mp.ChannelName {
if _, ok := rms.consumers[mp.ChannelName]; ok {
consumer := rms.consumers[mp.ChannelName]
msgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
if err != nil {
return err
}
messageID := UniqueID(msgID)
err = rms.consumers[index].Seek(messageID)
err = consumer.Seek(msgID)
if err != nil {
return err
}
return nil
}
}
return errors.New("msgStream seek fail")
}
@ -319,6 +333,7 @@ func (rms *RmqMsgStream) Seek(mp *msgstream.MsgPosition) error {
type RmqTtMsgStream struct {
RmqMsgStream
unsolvedBuf map[Consumer][]TsMsg
msgPositions map[Consumer]*internalpb.MsgPosition
unsolvedMutex *sync.Mutex
lastTimeStamp Timestamp
syncConsumer chan int
@ -330,24 +345,44 @@ func newRmqTtMsgStream(ctx context.Context, receiveBufSize int64, rmqBufSize int
if err != nil {
return nil, err
}
unsolvedBuf := make(map[Consumer][]TsMsg)
syncConsumer := make(chan int, 1)
msgPositions := make(map[Consumer]*internalpb.MsgPosition)
return &RmqTtMsgStream{
RmqMsgStream: *rmqMsgStream,
unsolvedBuf: unsolvedBuf,
msgPositions: msgPositions,
unsolvedMutex: &sync.Mutex{},
syncConsumer: syncConsumer,
}, nil
}
func (rtms *RmqTtMsgStream) addConsumer(consumer Consumer, channel string) {
if len(rtms.consumers) == 0 {
rtms.syncConsumer <- 1
}
rtms.consumers[channel] = consumer
rtms.unsolvedBuf[consumer] = make([]TsMsg, 0)
rtms.msgPositions[consumer] = &internalpb.MsgPosition{
ChannelName: channel,
MsgID: "",
Timestamp: rtms.lastTimeStamp,
}
rtms.consumerChannels = append(rtms.consumerChannels, channel)
}
func (rtms *RmqTtMsgStream) AsConsumer(channels []string,
groupName string) {
for i := 0; i < len(channels); i++ {
for _, channel := range channels {
if _, ok := rtms.consumers[channel]; ok {
continue
}
fn := func() error {
receiveChannel := make(chan client.ConsumerMessage, rtms.rmqBufSize)
pc, err := rtms.client.Subscribe(client.ConsumerOptions{
Topic: channels[i],
Topic: channel,
SubscriptionName: groupName,
MessageChannel: receiveChannel,
})
@ -355,22 +390,17 @@ func (rtms *RmqTtMsgStream) AsConsumer(channels []string,
return err
}
if pc == nil {
return errors.New("pulsar is not ready, consumer is nil")
return errors.New("RocksMQ is not ready, consumer is nil")
}
rtms.consumerLock.Lock()
if len(rtms.consumers) == 0 {
rtms.syncConsumer <- 1
}
rtms.consumers = append(rtms.consumers, pc)
rtms.unsolvedBuf[pc] = make([]TsMsg, 0)
rtms.consumerChannels = append(rtms.consumerChannels, channels[i])
rtms.addConsumer(pc, channel)
rtms.consumerLock.Unlock()
return nil
}
err := util.Retry(10, time.Millisecond*200, fn)
if err != nil {
errMsg := "Failed to create consumer " + channels[i] + ", error = " + err.Error()
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg)
}
}
@ -427,7 +457,8 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
continue
}
timeTickBuf := make([]TsMsg, 0)
msgPositions := make([]*msgstream.MsgPosition, 0)
startMsgPosition := make([]*internalpb.MsgPosition, 0)
endMsgPositions := make([]*internalpb.MsgPosition, 0)
rtms.unsolvedMutex.Lock()
for consumer, msgs := range rtms.unsolvedBuf {
if len(msgs) == 0 {
@ -448,19 +479,24 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
}
rtms.unsolvedBuf[consumer] = tempBuffer
startMsgPosition = append(startMsgPosition, rtms.msgPositions[consumer])
var newPos *internalpb.MsgPosition
if len(tempBuffer) > 0 {
msgPositions = append(msgPositions, &msgstream.MsgPosition{
newPos = &internalpb.MsgPosition{
ChannelName: tempBuffer[0].Position().ChannelName,
MsgID: tempBuffer[0].Position().MsgID,
Timestamp: timeStamp,
})
}
endMsgPositions = append(endMsgPositions, newPos)
} else {
msgPositions = append(msgPositions, &msgstream.MsgPosition{
newPos = &internalpb.MsgPosition{
ChannelName: timeTickMsg.Position().ChannelName,
MsgID: timeTickMsg.Position().MsgID,
Timestamp: timeStamp,
})
}
endMsgPositions = append(endMsgPositions, newPos)
}
rtms.msgPositions[consumer] = newPos
}
rtms.unsolvedMutex.Unlock()
@ -468,7 +504,8 @@ func (rtms *RmqTtMsgStream) bufMsgPackToChannel() {
BeginTs: rtms.lastTimeStamp,
EndTs: timeStamp,
Msgs: timeTickBuf,
StartPositions: msgPositions,
StartPositions: startMsgPosition,
EndPositions: endMsgPositions,
}
rtms.receiveBuf <- &msgPack
@ -524,36 +561,45 @@ func (rtms *RmqTtMsgStream) findTimeTick(consumer Consumer,
}
func (rtms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
var consumer Consumer
var messageID UniqueID
for index, channel := range rtms.consumerChannels {
if filepath.Base(channel) == filepath.Base(mp.ChannelName) {
consumer = rtms.consumers[index]
if len(mp.MsgID) == 0 {
messageID = -1
break
return errors.New("when msgID's length equal to 0, please use AsConsumer interface")
}
var consumer Consumer
var err error
var hasWatched bool
seekChannel := mp.ChannelName
subName := mp.MsgGroup
rtms.consumerLock.Lock()
defer rtms.consumerLock.Unlock()
consumer, hasWatched = rtms.consumers[seekChannel]
if hasWatched {
return errors.New("the channel should has been subscribed")
}
receiveChannel := make(chan client.ConsumerMessage, rtms.rmqBufSize)
consumer, err = rtms.client.Subscribe(client.ConsumerOptions{
Topic: seekChannel,
SubscriptionName: subName,
MessageChannel: receiveChannel,
})
if err != nil {
return err
}
if consumer == nil {
return errors.New("RocksMQ is not ready, consumer is nil")
}
seekMsgID, err := strconv.ParseInt(mp.MsgID, 10, 64)
if err != nil {
return err
}
messageID = seekMsgID
break
}
}
consumer.Seek(seekMsgID)
rtms.addConsumer(consumer, seekChannel)
if consumer != nil {
err := (consumer).Seek(messageID)
if err != nil {
return err
}
//TODO: Is this right?
if messageID == 0 {
if len(consumer.Chan()) == 0 {
return nil
}
rtms.unsolvedMutex.Lock()
rtms.unsolvedBuf[consumer] = make([]TsMsg, 0)
for {
select {
case <-rtms.ctx.Done():
@ -574,7 +620,6 @@ func (rtms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
}
if tsMsg.Type() == commonpb.MsgType_TimeTick {
if tsMsg.BeginTs() >= mp.Timestamp {
rtms.unsolvedMutex.Unlock()
return nil
}
continue
@ -588,9 +633,6 @@ func (rtms *RmqTtMsgStream) Seek(mp *msgstream.MsgPosition) error {
}
}
}
}
return errors.New("msgStream seek fail")
}
func checkTimeTickMsg(msg map[Consumer]Timestamp,

3
internal/proto/internal.proto
View File

@ -209,5 +209,6 @@ message QueryNodeStats {
message MsgPosition {
string channel_name = 1;
string msgID = 2;
uint64 timestamp = 3;
string msgGroup = 3;
uint64 timestamp = 4;
}

203
internal/proto/internalpb/internal.pb.go
View File

@ -1802,7 +1802,8 @@ func (m *QueryNodeStats) GetFieldStats() []*FieldStats {
type MsgPosition struct {
ChannelName string `protobuf:"bytes,1,opt,name=channel_name,json=channelName,proto3" json:"channel_name,omitempty"`
MsgID string `protobuf:"bytes,2,opt,name=msgID,proto3" json:"msgID,omitempty"`
Timestamp uint64 `protobuf:"varint,3,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
MsgGroup string `protobuf:"bytes,3,opt,name=msgGroup,proto3" json:"msgGroup,omitempty"`
Timestamp uint64 `protobuf:"varint,4,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1847,6 +1848,13 @@ func (m *MsgPosition) GetMsgID() string {
return ""
}
func (m *MsgPosition) GetMsgGroup() string {
if m != nil {
return m.MsgGroup
}
return ""
}
func (m *MsgPosition) GetTimestamp() uint64 {
if m != nil {
return m.Timestamp
@ -1890,101 +1898,102 @@ func init() {
func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
var fileDescriptor_41f4a519b878ee3b = []byte{
// 1524 bytes of a gzipped FileDescriptorProto
// 1539 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0x6d, 0x27, 0xb6, 0xdf, 0x3a, 0xa9, 0xbb, 0xfd, 0xda, 0xd0, 0x94, 0xba, 0xcb, 0x57,
0xa0, 0x22, 0xa9, 0x52, 0x84, 0x10, 0x97, 0xb6, 0x89, 0x69, 0xb0, 0xda, 0x44, 0x61, 0x9d, 0x56,
0x82, 0xcb, 0x6a, 0xbc, 0x3b, 0xb1, 0xa7, 0xdd, 0x0f, 0x77, 0x66, 0xb6, 0xa9, 0x73, 0xe6, 0x86,
0xe0, 0x80, 0xc4, 0x3f, 0xc0, 0x1f, 0xc0, 0x99, 0x13, 0x20, 0x4e, 0x48, 0xdc, 0x91, 0x90, 0xf8,
0x4b, 0x38, 0xa1, 0xf9, 0xd8, 0xf5, 0x47, 0x9d, 0x34, 0x35, 0x54, 0x08, 0xc1, 0xcd, 0xf3, 0x9b,
0xb7, 0x6f, 0xe6, 0xf7, 0x7b, 0xef, 0xcd, 0x9b, 0x31, 0x2c, 0x92, 0x98, 0x63, 0x1a, 0xa3, 0x70,
0xb5, 0x4f, 0x13, 0x9e, 0x58, 0xe7, 0x22, 0x12, 0x3e, 0x4e, 0x99, 0x1a, 0xad, 0x66, 0x93, 0x2f,
0xd7, 0xfc, 0x24, 0x8a, 0x92, 0x58, 0xc1, 0xce, 0xf7, 0x06, 0x2c, 0x6c, 0x26, 0x51, 0x3f, 0x89,
0x71, 0xcc, 0x5b, 0xf1, 0x7e, 0x62, 0x9d, 0x87, 0xf9, 0x38, 0x09, 0x70, 0xab, 0x69, 0x1b, 0x0d,
0x63, 0xa5, 0xe8, 0xea, 0x91, 0x65, 0x41, 0x89, 0x26, 0x21, 0xb6, 0x0b, 0x0d, 0x63, 0xa5, 0xea,
0xca, 0xdf, 0xd6, 0x0d, 0x00, 0xc6, 0x11, 0xc7, 0x9e, 0x9f, 0x04, 0xd8, 0x2e, 0x36, 0x8c, 0x95,
0xc5, 0xf5, 0xc6, 0xea, 0xd4, 0x75, 0x57, 0xdb, 0xc2, 0x70, 0x33, 0x09, 0xb0, 0x5b, 0x65, 0xd9,
0x4f, 0xeb, 0x26, 0x00, 0x7e, 0xc2, 0x29, 0xf2, 0x48, 0xbc, 0x9f, 0xd8, 0xa5, 0x46, 0x71, 0xc5,
0x5c, 0xbf, 0x32, 0xee, 0x40, 0x6f, 0xf7, 0x0e, 0x1e, 0xdc, 0x47, 0x61, 0x8a, 0x77, 0x11, 0xa1,
0x6e, 0x55, 0x7e, 0x24, 0xb6, 0xeb, 0xfc, 0x66, 0xc0, 0xa9, 0x9c, 0x80, 0x5c, 0x83, 0x59, 0x1f,
0xc0, 0x9c, 0x5c, 0x42, 0x32, 0x30, 0xd7, 0x5f, 0x3b, 0x62, 0x47, 0x63, 0xbc, 0x5d, 0xf5, 0x89,
0x75, 0x0f, 0xce, 0xb0, 0xb4, 0xe3, 0x67, 0x53, 0x9e, 0x44, 0x99, 0x5d, 0x90, 0x5b, 0x3b, 0x99,
0x27, 0x6b, 0xd4, 0x81, 0xde, 0xd2, 0x75, 0x98, 0x17, 0x9e, 0x52, 0x26, 0x55, 0x32, 0xd7, 0x2f,
0x4e, 0x25, 0xd9, 0x96, 0x26, 0xae, 0x36, 0x75, 0x2e, 0xc2, 0xd2, 0x16, 0xe6, 0x13, 0xec, 0x5c,
0xfc, 0x28, 0xc5, 0x8c, 0xeb, 0xc9, 0x3d, 0x12, 0xe1, 0x3d, 0xe2, 0x3f, 0xdc, 0xec, 0xa1, 0x38,
0xc6, 0x61, 0x36, 0x79, 0x09, 0x2e, 0x6e, 0x61, 0xf9, 0x01, 0x61, 0x9c, 0xf8, 0x6c, 0x62, 0xfa,
0x1c, 0x9c, 0xd9, 0xc2, 0xbc, 0x19, 0x4c, 0xc0, 0xf7, 0xa1, 0xb2, 0x23, 0x82, 0x2d, 0xd2, 0xe0,
0x3d, 0x28, 0xa3, 0x20, 0xa0, 0x98, 0x31, 0xad, 0xe2, 0xf2, 0xd4, 0x1d, 0xdf, 0x52, 0x36, 0x6e,
0x66, 0x3c, 0x2d, 0x4d, 0x9c, 0x07, 0x00, 0xad, 0x98, 0xf0, 0x5d, 0x44, 0x51, 0xc4, 0x8e, 0x4c,
0xb0, 0x26, 0xd4, 0x18, 0x47, 0x94, 0x7b, 0x7d, 0x69, 0xa7, 0x25, 0x3f, 0x41, 0x36, 0x98, 0xf2,
0x33, 0xe5, 0xdd, 0xf9, 0x04, 0xa0, 0xcd, 0x29, 0x89, 0xbb, 0x77, 0x09, 0xe3, 0x62, 0xad, 0xc7,
0xc2, 0x4e, 0x90, 0x28, 0xae, 0x54, 0x5d, 0x3d, 0x1a, 0x09, 0x47, 0xe1, 0xe4, 0xe1, 0xb8, 0x01,
0x66, 0x26, 0xf7, 0x36, 0xeb, 0x5a, 0xd7, 0xa0, 0xd4, 0x41, 0x0c, 0x1f, 0x2b, 0xcf, 0x36, 0xeb,
0x6e, 0x20, 0x86, 0x5d, 0x69, 0xe9, 0xfc, 0x6e, 0xc0, 0x85, 0x4d, 0x8a, 0x65, 0xf2, 0x87, 0x21,
0xf6, 0x39, 0x49, 0x62, 0xad, 0xfd, 0xf3, 0x7b, 0xb3, 0x2e, 0x40, 0x39, 0xe8, 0x78, 0x31, 0x8a,
0x32, 0xb1, 0xe7, 0x83, 0xce, 0x0e, 0x8a, 0xb0, 0xf5, 0x06, 0x2c, 0xfa, 0xb9, 0x7f, 0x81, 0xc8,
0x9c, 0xab, 0xba, 0x13, 0xa8, 0x08, 0x55, 0xd0, 0x69, 0x35, 0xed, 0x92, 0x0c, 0x83, 0xfc, 0x6d,
0x39, 0x50, 0x1b, 0x5a, 0xb5, 0x9a, 0xf6, 0x9c, 0x9c, 0x1b, 0xc3, 0x84, 0xa8, 0xcc, 0xef, 0xe1,
0x08, 0xd9, 0xf3, 0x0d, 0x63, 0xa5, 0xe6, 0xea, 0x91, 0xf3, 0x93, 0x01, 0xe7, 0x9a, 0x34, 0xe9,
0xff, 0x9b, 0xc9, 0x39, 0x5f, 0x14, 0xe0, 0xbc, 0x8a, 0xd1, 0x2e, 0xa2, 0x9c, 0xbc, 0x20, 0x16,
0x6f, 0xc2, 0xa9, 0xe1, 0xaa, 0xca, 0x60, 0x3a, 0x8d, 0xd7, 0x61, 0xb1, 0x9f, 0xed, 0x43, 0xd9,
0x95, 0xa4, 0xdd, 0x42, 0x8e, 0x8e, 0xb1, 0x9d, 0x3b, 0x86, 0xed, 0xfc, 0x94, 0x50, 0x36, 0xc0,
0xcc, 0x1d, 0xb5, 0x9a, 0x76, 0x59, 0x9a, 0x8c, 0x42, 0xce, 0xe7, 0x05, 0x38, 0x2b, 0x82, 0xfa,
0xbf, 0x1a, 0x42, 0x8d, 0x1f, 0x0a, 0x60, 0xa9, 0xec, 0x68, 0xc5, 0x01, 0x7e, 0xf2, 0x4f, 0x6a,
0x71, 0x09, 0x60, 0x9f, 0xe0, 0x30, 0x18, 0xd5, 0xa1, 0x2a, 0x91, 0xbf, 0xa4, 0x81, 0x0d, 0x65,
0xe9, 0x24, 0xe7, 0x9f, 0x0d, 0xc5, 0xf9, 0xac, 0x7a, 0xb5, 0x3e, 0x9f, 0x2b, 0x27, 0x3e, 0x9f,
0xe5, 0x67, 0xfa, 0x7c, 0xfe, 0xb6, 0x08, 0x0b, 0xad, 0x98, 0x61, 0xca, 0xff, 0xcb, 0x89, 0x64,
0x2d, 0x43, 0x95, 0xe1, 0x6e, 0x24, 0xae, 0x0c, 0x4d, 0xbb, 0x22, 0xe7, 0x87, 0x80, 0x98, 0xf5,
0x55, 0x6b, 0x6e, 0x35, 0xed, 0xaa, 0x0a, 0x6d, 0x0e, 0x58, 0xaf, 0x00, 0x70, 0x12, 0x61, 0xc6,
0x51, 0xd4, 0x67, 0x36, 0x34, 0x8a, 0x2b, 0x25, 0x77, 0x04, 0x11, 0xe7, 0x33, 0x4d, 0x0e, 0x5a,
0x4d, 0x66, 0x9b, 0x8d, 0xa2, 0x68, 0xb0, 0x6a, 0x64, 0xbd, 0x0b, 0x15, 0x9a, 0x1c, 0x78, 0x01,
0xe2, 0xc8, 0xae, 0xc9, 0xe0, 0x2d, 0x4d, 0x15, 0x7b, 0x23, 0x4c, 0x3a, 0x6e, 0x99, 0x26, 0x07,
0x4d, 0xc4, 0x91, 0xf3, 0x5d, 0x01, 0x16, 0xda, 0x18, 0x51, 0xbf, 0x37, 0x7b, 0xc0, 0xde, 0x82,
0x3a, 0xc5, 0x2c, 0x0d, 0xb9, 0x37, 0xa4, 0xa5, 0x22, 0x77, 0x4a, 0xe1, 0x9b, 0x39, 0xb9, 0x4c,
0xf2, 0xe2, 0x31, 0x92, 0x97, 0xa6, 0x48, 0xee, 0x40, 0x6d, 0x44, 0x5f, 0x66, 0xcf, 0x49, 0xea,
0x63, 0x98, 0x55, 0x87, 0x62, 0xc0, 0x42, 0x19, 0xb1, 0xaa, 0x2b, 0x7e, 0x5a, 0x57, 0xe1, 0x74,
0x3f, 0x44, 0x3e, 0xee, 0x25, 0x61, 0x80, 0xa9, 0xd7, 0xa5, 0x49, 0xda, 0x97, 0xe1, 0xaa, 0xb9,
0xf5, 0x91, 0x89, 0x2d, 0x81, 0x5b, 0x6b, 0x30, 0xf7, 0x28, 0xc5, 0x74, 0x20, 0xe3, 0x75, 0xac,
0x78, 0xca, 0xce, 0xf9, 0xd5, 0x18, 0x4a, 0x27, 0x58, 0xb2, 0x19, 0xa4, 0x9b, 0xe5, 0xa6, 0x32,
0x55, 0xef, 0xe2, 0x74, 0xbd, 0x2f, 0x83, 0x19, 0x61, 0x4e, 0x89, 0xef, 0xf1, 0x41, 0x3f, 0x2b,
0x03, 0x50, 0xd0, 0xde, 0xa0, 0x2f, 0x6b, 0xa0, 0x47, 0xb8, 0x12, 0xb4, 0xe6, 0xca, 0xdf, 0xce,
0x2f, 0x06, 0x2c, 0x34, 0x71, 0x88, 0x39, 0x9e, 0x3d, 0x27, 0xa6, 0xd4, 0x6a, 0x61, 0x6a, 0xad,
0x8e, 0x15, 0x43, 0xf1, 0xf8, 0x62, 0x28, 0x3d, 0x55, 0x0c, 0x57, 0xa0, 0xd6, 0xa7, 0x24, 0x42,
0x74, 0xe0, 0x3d, 0xc4, 0x83, 0x2c, 0x2f, 0x4c, 0x8d, 0xdd, 0xc1, 0x03, 0xe6, 0x7c, 0x63, 0x40,
0xe5, 0x76, 0x98, 0xb2, 0xde, 0x4c, 0xb7, 0xba, 0xf1, 0x52, 0x2e, 0x4c, 0x96, 0xf2, 0x64, 0xee,
0x16, 0x9f, 0x91, 0xbb, 0x7b, 0xa8, 0xab, 0x83, 0x30, 0x86, 0x39, 0x7f, 0x18, 0x50, 0xbd, 0x9b,
0xa0, 0x40, 0xf6, 0x9d, 0xbf, 0x7d, 0x97, 0xcb, 0x30, 0x6c, 0x1d, 0x99, 0xc6, 0xc3, 0x5e, 0x32,
0xd2, 0x13, 0x4a, 0xe3, 0x3d, 0xe1, 0x32, 0x98, 0x44, 0x6c, 0xc8, 0xeb, 0x23, 0xde, 0x53, 0xe2,
0x56, 0x5d, 0x90, 0xd0, 0xae, 0x40, 0x44, 0xd3, 0xc8, 0x0c, 0x64, 0xd3, 0x98, 0x3f, 0x71, 0xd3,
0xd0, 0x4e, 0x64, 0xd3, 0xf8, 0xb1, 0x00, 0x76, 0x5b, 0x6d, 0x76, 0xf8, 0xa6, 0xb9, 0xd7, 0x0f,
0xe4, 0xd3, 0x6a, 0x19, 0xaa, 0xed, 0x9c, 0x99, 0x7a, 0x52, 0x0c, 0x01, 0x91, 0x1f, 0xdb, 0x38,
0x4a, 0xe8, 0xa0, 0x4d, 0x0e, 0xb1, 0x26, 0x3e, 0x82, 0x08, 0x6e, 0x3b, 0x69, 0xe4, 0x26, 0x07,
0x4c, 0x87, 0x26, 0x1b, 0x0a, 0x6e, 0xbe, 0x6c, 0xf5, 0x9e, 0x48, 0x27, 0xc9, 0xbc, 0xe4, 0x82,
0x82, 0xc4, 0x3b, 0xc0, 0x5a, 0x82, 0x0a, 0x8e, 0x03, 0x35, 0x3b, 0x27, 0x67, 0xcb, 0x38, 0x0e,
0xe4, 0x54, 0x0b, 0x16, 0xf5, 0x5b, 0x26, 0x61, 0x32, 0x84, 0xf2, 0xd0, 0x31, 0xd7, 0x9d, 0x23,
0x1e, 0x90, 0xdb, 0xac, 0xbb, 0xab, 0x2d, 0xdd, 0x05, 0xf5, 0x9c, 0xd1, 0x43, 0xeb, 0x43, 0xa8,
0x89, 0x55, 0x72, 0x47, 0xe5, 0x13, 0x3b, 0x32, 0x71, 0x1c, 0x64, 0x03, 0xe7, 0x2b, 0x03, 0x4e,
0x3f, 0x25, 0xe1, 0x0c, 0x79, 0x74, 0x07, 0x2a, 0x6d, 0xdc, 0x15, 0x2e, 0xb2, 0x17, 0xda, 0xda,
0x51, 0x0f, 0xfe, 0x23, 0x02, 0xe6, 0xe6, 0x0e, 0x9c, 0x07, 0x79, 0x58, 0x65, 0xfd, 0x89, 0x97,
0xae, 0x38, 0x54, 0x82, 0x17, 0x50, 0x88, 0xce, 0x67, 0x86, 0x78, 0x85, 0x06, 0xf8, 0x89, 0x5c,
0xfa, 0xa9, 0xc4, 0x34, 0x66, 0x49, 0x4c, 0xeb, 0x1a, 0x9c, 0x8d, 0xd3, 0xc8, 0xa3, 0x38, 0x44,
0x1c, 0x07, 0x9e, 0x5e, 0x8d, 0xe9, 0xd5, 0xad, 0x38, 0x8d, 0x5c, 0x35, 0xa5, 0x69, 0x32, 0xe7,
0x4b, 0x03, 0xe0, 0xb6, 0xa8, 0x1e, 0xb5, 0x8d, 0xc9, 0xe3, 0xc1, 0x38, 0xfe, 0x4a, 0x56, 0x18,
0x2f, 0xbf, 0x8d, 0xac, 0xfc, 0x98, 0x8c, 0x47, 0x71, 0x1a, 0x87, 0x3c, 0x1e, 0x43, 0xf2, 0xba,
0x42, 0x55, 0x0c, 0xbe, 0x36, 0xa0, 0x36, 0x12, 0x2a, 0x36, 0x2e, 0xa3, 0x31, 0x79, 0x52, 0xc8,
0x7e, 0x21, 0xaa, 0xc7, 0x63, 0x23, 0x05, 0x15, 0x0d, 0x0b, 0x6a, 0x09, 0x2a, 0x52, 0x92, 0x91,
0x8a, 0x8a, 0x75, 0x45, 0x5d, 0x85, 0xd3, 0x14, 0xfb, 0x38, 0xe6, 0xe1, 0xc0, 0x8b, 0x92, 0x80,
0xec, 0x13, 0x1c, 0xc8, 0xba, 0xaa, 0xb8, 0xf5, 0x6c, 0x62, 0x5b, 0xe3, 0xce, 0xcf, 0x06, 0x2c,
0x7e, 0x2c, 0xda, 0xe8, 0x4e, 0x12, 0x60, 0xb5, 0xb3, 0xe7, 0x4f, 0x89, 0x9b, 0x92, 0x8b, 0x96,
0x47, 0xa5, 0xeb, 0xab, 0xcf, 0x4e, 0x57, 0xe6, 0x56, 0x98, 0x4e, 0x51, 0x21, 0xb1, 0xba, 0x66,
0x9f, 0x44, 0xe2, 0x61, 0x60, 0x5d, 0x75, 0x39, 0x57, 0x12, 0x07, 0x60, 0x8e, 0xd4, 0xa5, 0x68,
0x49, 0xba, 0x7f, 0xa9, 0xb6, 0x67, 0xc8, 0xf3, 0xd6, 0xd4, 0x98, 0x3c, 0x71, 0xcf, 0xc2, 0x5c,
0xc4, 0xba, 0xf9, 0x2d, 0x49, 0x0d, 0x44, 0x64, 0xf2, 0xce, 0x26, 0xb5, 0x2d, 0xb9, 0x43, 0xe0,
0xed, 0xf7, 0xa1, 0x9a, 0xff, 0xc7, 0x66, 0xd5, 0xa1, 0xd6, 0x8a, 0x09, 0x27, 0x28, 0x24, 0x87,
0x24, 0xee, 0xd6, 0x5f, 0xb2, 0x4c, 0x28, 0x7f, 0x84, 0x51, 0xc8, 0x7b, 0x83, 0xba, 0x61, 0xd5,
0xa0, 0x72, 0xab, 0x13, 0x27, 0x34, 0x42, 0x61, 0xbd, 0xb0, 0xd1, 0xfc, 0x74, 0xa3, 0x4b, 0x78,
0x2f, 0xed, 0x08, 0x11, 0xd7, 0x0e, 0x49, 0x18, 0x92, 0x43, 0x8e, 0xfd, 0xde, 0x9a, 0x62, 0xf9,
0x4e, 0x40, 0x18, 0xa7, 0xa4, 0x93, 0x72, 0x1c, 0xac, 0x65, 0x5c, 0xd7, 0x24, 0xf5, 0x7c, 0xd8,
0xef, 0x74, 0xe6, 0x25, 0x72, 0xfd, 0xcf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x4c, 0x6c, 0x9f,
0x88, 0x14, 0x00, 0x00,
0x1b, 0x7f, 0xd7, 0x76, 0x62, 0xfb, 0x59, 0x27, 0x75, 0xb7, 0x5f, 0x9b, 0x36, 0x7d, 0xeb, 0xee,
0xfb, 0x02, 0x81, 0x8a, 0xa4, 0x4a, 0x11, 0x42, 0x5c, 0xda, 0x26, 0xa6, 0xc1, 0x6a, 0x13, 0x85,
0x75, 0x5a, 0x09, 0x2e, 0xab, 0xf1, 0xee, 0xc4, 0x9e, 0x76, 0x3f, 0xdc, 0x99, 0xd9, 0xa6, 0xce,
0x89, 0x03, 0x37, 0x04, 0x07, 0x24, 0xfe, 0x01, 0xfe, 0x00, 0xce, 0x9c, 0x00, 0x71, 0x42, 0xe2,
0x8e, 0x84, 0xc4, 0x5f, 0xc2, 0x09, 0xcd, 0xc7, 0xae, 0x3f, 0xea, 0xa4, 0xa9, 0xa1, 0x42, 0x08,
0x6e, 0x9e, 0xdf, 0xf3, 0xec, 0x33, 0xf3, 0xfc, 0x9e, 0xaf, 0x19, 0xc3, 0x22, 0x89, 0x39, 0xa6,
0x31, 0x0a, 0x57, 0xfb, 0x34, 0xe1, 0x89, 0x75, 0x2e, 0x22, 0xe1, 0x93, 0x94, 0xa9, 0xd5, 0x6a,
0x26, 0xbc, 0x58, 0xf3, 0x93, 0x28, 0x4a, 0x62, 0x05, 0x3b, 0xdf, 0x1a, 0xb0, 0xb0, 0x99, 0x44,
0xfd, 0x24, 0xc6, 0x31, 0x6f, 0xc5, 0xfb, 0x89, 0x75, 0x1e, 0xe6, 0xe3, 0x24, 0xc0, 0xad, 0xa6,
0x6d, 0x34, 0x8c, 0x95, 0xa2, 0xab, 0x57, 0x96, 0x05, 0x25, 0x9a, 0x84, 0xd8, 0x2e, 0x34, 0x8c,
0x95, 0xaa, 0x2b, 0x7f, 0x5b, 0x37, 0x01, 0x18, 0x47, 0x1c, 0x7b, 0x7e, 0x12, 0x60, 0xbb, 0xd8,
0x30, 0x56, 0x16, 0xd7, 0x1b, 0xab, 0x53, 0xf7, 0x5d, 0x6d, 0x0b, 0xc5, 0xcd, 0x24, 0xc0, 0x6e,
0x95, 0x65, 0x3f, 0xad, 0x5b, 0x00, 0xf8, 0x29, 0xa7, 0xc8, 0x23, 0xf1, 0x7e, 0x62, 0x97, 0x1a,
0xc5, 0x15, 0x73, 0xfd, 0xea, 0xb8, 0x01, 0x7d, 0xdc, 0xbb, 0x78, 0xf0, 0x00, 0x85, 0x29, 0xde,
0x45, 0x84, 0xba, 0x55, 0xf9, 0x91, 0x38, 0xae, 0xf3, 0x8b, 0x01, 0xa7, 0x72, 0x07, 0xe4, 0x1e,
0xcc, 0x7a, 0x17, 0xe6, 0xe4, 0x16, 0xd2, 0x03, 0x73, 0xfd, 0xff, 0x47, 0x9c, 0x68, 0xcc, 0x6f,
0x57, 0x7d, 0x62, 0xdd, 0x87, 0x33, 0x2c, 0xed, 0xf8, 0x99, 0xc8, 0x93, 0x28, 0xb3, 0x0b, 0xf2,
0x68, 0x27, 0xb3, 0x64, 0x8d, 0x1a, 0xd0, 0x47, 0xba, 0x01, 0xf3, 0xc2, 0x52, 0xca, 0x24, 0x4b,
0xe6, 0xfa, 0xa5, 0xa9, 0x4e, 0xb6, 0xa5, 0x8a, 0xab, 0x55, 0x9d, 0x4b, 0xb0, 0xb4, 0x85, 0xf9,
0x84, 0x77, 0x2e, 0x7e, 0x9c, 0x62, 0xc6, 0xb5, 0x70, 0x8f, 0x44, 0x78, 0x8f, 0xf8, 0x8f, 0x36,
0x7b, 0x28, 0x8e, 0x71, 0x98, 0x09, 0x2f, 0xc3, 0xa5, 0x2d, 0x2c, 0x3f, 0x20, 0x8c, 0x13, 0x9f,
0x4d, 0x88, 0xcf, 0xc1, 0x99, 0x2d, 0xcc, 0x9b, 0xc1, 0x04, 0xfc, 0x00, 0x2a, 0x3b, 0x22, 0xd8,
0x22, 0x0d, 0xde, 0x86, 0x32, 0x0a, 0x02, 0x8a, 0x19, 0xd3, 0x2c, 0x2e, 0x4f, 0x3d, 0xf1, 0x6d,
0xa5, 0xe3, 0x66, 0xca, 0xd3, 0xd2, 0xc4, 0x79, 0x08, 0xd0, 0x8a, 0x09, 0xdf, 0x45, 0x14, 0x45,
0xec, 0xc8, 0x04, 0x6b, 0x42, 0x8d, 0x71, 0x44, 0xb9, 0xd7, 0x97, 0x7a, 0x9a, 0xf2, 0x13, 0x64,
0x83, 0x29, 0x3f, 0x53, 0xd6, 0x9d, 0x0f, 0x01, 0xda, 0x9c, 0x92, 0xb8, 0x7b, 0x8f, 0x30, 0x2e,
0xf6, 0x7a, 0x22, 0xf4, 0x84, 0x13, 0xc5, 0x95, 0xaa, 0xab, 0x57, 0x23, 0xe1, 0x28, 0x9c, 0x3c,
0x1c, 0x37, 0xc1, 0xcc, 0xe8, 0xde, 0x66, 0x5d, 0xeb, 0x3a, 0x94, 0x3a, 0x88, 0xe1, 0x63, 0xe9,
0xd9, 0x66, 0xdd, 0x0d, 0xc4, 0xb0, 0x2b, 0x35, 0x9d, 0x5f, 0x0d, 0xb8, 0xb0, 0x49, 0xb1, 0x4c,
0xfe, 0x30, 0xc4, 0x3e, 0x27, 0x49, 0xac, 0xb9, 0x7f, 0x71, 0x6b, 0xd6, 0x05, 0x28, 0x07, 0x1d,
0x2f, 0x46, 0x51, 0x46, 0xf6, 0x7c, 0xd0, 0xd9, 0x41, 0x11, 0xb6, 0x5e, 0x85, 0x45, 0x3f, 0xb7,
0x2f, 0x10, 0x99, 0x73, 0x55, 0x77, 0x02, 0x15, 0xa1, 0x0a, 0x3a, 0xad, 0xa6, 0x5d, 0x92, 0x61,
0x90, 0xbf, 0x2d, 0x07, 0x6a, 0x43, 0xad, 0x56, 0xd3, 0x9e, 0x93, 0xb2, 0x31, 0x4c, 0x90, 0xca,
0xfc, 0x1e, 0x8e, 0x90, 0x3d, 0xdf, 0x30, 0x56, 0x6a, 0xae, 0x5e, 0x39, 0x3f, 0x18, 0x70, 0xae,
0x49, 0x93, 0xfe, 0xdf, 0xd9, 0x39, 0xe7, 0xb3, 0x02, 0x9c, 0x57, 0x31, 0xda, 0x45, 0x94, 0x93,
0x97, 0xe4, 0xc5, 0x6b, 0x70, 0x6a, 0xb8, 0xab, 0x52, 0x98, 0xee, 0xc6, 0x2b, 0xb0, 0xd8, 0xcf,
0xce, 0xa1, 0xf4, 0x4a, 0x52, 0x6f, 0x21, 0x47, 0xc7, 0xbc, 0x9d, 0x3b, 0xc6, 0xdb, 0xf9, 0x29,
0xa1, 0x6c, 0x80, 0x99, 0x1b, 0x6a, 0x35, 0xed, 0xb2, 0x54, 0x19, 0x85, 0x9c, 0x4f, 0x0b, 0x70,
0x56, 0x04, 0xf5, 0x5f, 0x36, 0x04, 0x1b, 0xdf, 0x15, 0xc0, 0x52, 0xd9, 0xd1, 0x8a, 0x03, 0xfc,
0xf4, 0xaf, 0xe4, 0xe2, 0x32, 0xc0, 0x3e, 0xc1, 0x61, 0x30, 0xca, 0x43, 0x55, 0x22, 0x7f, 0x88,
0x03, 0x1b, 0xca, 0xd2, 0x48, 0xee, 0x7f, 0xb6, 0x14, 0xfd, 0x59, 0xcd, 0x6a, 0xdd, 0x9f, 0x2b,
0x27, 0xee, 0xcf, 0xf2, 0x33, 0xdd, 0x9f, 0xbf, 0x2e, 0xc2, 0x42, 0x2b, 0x66, 0x98, 0xf2, 0x7f,
0x72, 0x22, 0x59, 0xcb, 0x50, 0x65, 0xb8, 0x1b, 0x89, 0x2b, 0x43, 0xd3, 0xae, 0x48, 0xf9, 0x10,
0x10, 0x52, 0x5f, 0x8d, 0xe6, 0x56, 0xd3, 0xae, 0xaa, 0xd0, 0xe6, 0x80, 0xf5, 0x5f, 0x00, 0x4e,
0x22, 0xcc, 0x38, 0x8a, 0xfa, 0xcc, 0x86, 0x46, 0x71, 0xa5, 0xe4, 0x8e, 0x20, 0xa2, 0x3f, 0xd3,
0xe4, 0xa0, 0xd5, 0x64, 0xb6, 0xd9, 0x28, 0x8a, 0x01, 0xab, 0x56, 0xd6, 0x5b, 0x50, 0xa1, 0xc9,
0x81, 0x17, 0x20, 0x8e, 0xec, 0x9a, 0x0c, 0xde, 0xd2, 0x54, 0xb2, 0x37, 0xc2, 0xa4, 0xe3, 0x96,
0x69, 0x72, 0xd0, 0x44, 0x1c, 0x39, 0xdf, 0x14, 0x60, 0xa1, 0x8d, 0x11, 0xf5, 0x7b, 0xb3, 0x07,
0xec, 0x75, 0xa8, 0x53, 0xcc, 0xd2, 0x90, 0x7b, 0x43, 0xb7, 0x54, 0xe4, 0x4e, 0x29, 0x7c, 0x33,
0x77, 0x2e, 0xa3, 0xbc, 0x78, 0x0c, 0xe5, 0xa5, 0x29, 0x94, 0x3b, 0x50, 0x1b, 0xe1, 0x97, 0xd9,
0x73, 0xd2, 0xf5, 0x31, 0xcc, 0xaa, 0x43, 0x31, 0x60, 0xa1, 0x8c, 0x58, 0xd5, 0x15, 0x3f, 0xad,
0x6b, 0x70, 0xba, 0x1f, 0x22, 0x1f, 0xf7, 0x92, 0x30, 0xc0, 0xd4, 0xeb, 0xd2, 0x24, 0xed, 0xcb,
0x70, 0xd5, 0xdc, 0xfa, 0x88, 0x60, 0x4b, 0xe0, 0xd6, 0x1a, 0xcc, 0x3d, 0x4e, 0x31, 0x1d, 0xc8,
0x78, 0x1d, 0x4b, 0x9e, 0xd2, 0x73, 0x7e, 0x36, 0x86, 0xd4, 0x09, 0x2f, 0xd9, 0x0c, 0xd4, 0xcd,
0x72, 0x53, 0x99, 0xca, 0x77, 0x71, 0x3a, 0xdf, 0x57, 0xc0, 0x8c, 0x30, 0xa7, 0xc4, 0xf7, 0xf8,
0xa0, 0x9f, 0x95, 0x01, 0x28, 0x68, 0x6f, 0xd0, 0x97, 0x35, 0xd0, 0x23, 0x5c, 0x11, 0x5a, 0x73,
0xe5, 0x6f, 0xe7, 0x27, 0x03, 0x16, 0x9a, 0x38, 0xc4, 0x1c, 0xcf, 0x9e, 0x13, 0x53, 0x6a, 0xb5,
0x30, 0xb5, 0x56, 0xc7, 0x8a, 0xa1, 0x78, 0x7c, 0x31, 0x94, 0x9e, 0x29, 0x86, 0xab, 0x50, 0xeb,
0x53, 0x12, 0x21, 0x3a, 0xf0, 0x1e, 0xe1, 0x41, 0x96, 0x17, 0xa6, 0xc6, 0xee, 0xe2, 0x01, 0x73,
0xbe, 0x32, 0xa0, 0x72, 0x27, 0x4c, 0x59, 0x6f, 0xa6, 0x5b, 0xdd, 0x78, 0x29, 0x17, 0x26, 0x4b,
0x79, 0x32, 0x77, 0x8b, 0xcf, 0xc9, 0xdd, 0x3d, 0xd4, 0xd5, 0x41, 0x18, 0xc3, 0x9c, 0xdf, 0x0c,
0xa8, 0xde, 0x4b, 0x50, 0x20, 0xe7, 0xce, 0x9f, 0x7e, 0xca, 0x65, 0x18, 0x8e, 0x8e, 0x8c, 0xe3,
0xe1, 0x2c, 0x19, 0x99, 0x09, 0xa5, 0xf1, 0x99, 0x70, 0x05, 0x4c, 0x22, 0x0e, 0xe4, 0xf5, 0x11,
0xef, 0x29, 0x72, 0xab, 0x2e, 0x48, 0x68, 0x57, 0x20, 0x62, 0x68, 0x64, 0x0a, 0x72, 0x68, 0xcc,
0x9f, 0x78, 0x68, 0x68, 0x23, 0x72, 0x68, 0x7c, 0x5f, 0x00, 0xbb, 0xad, 0x0e, 0x3b, 0x7c, 0xd3,
0xdc, 0xef, 0x07, 0xf2, 0x69, 0xb5, 0x0c, 0xd5, 0x76, 0xee, 0x99, 0x7a, 0x52, 0x0c, 0x01, 0x91,
0x1f, 0xdb, 0x38, 0x4a, 0xe8, 0xa0, 0x4d, 0x0e, 0xb1, 0x76, 0x7c, 0x04, 0x11, 0xbe, 0xed, 0xa4,
0x91, 0x9b, 0x1c, 0x30, 0x1d, 0x9a, 0x6c, 0x29, 0x7c, 0xf3, 0xe5, 0xa8, 0xf7, 0x44, 0x3a, 0x49,
0xcf, 0x4b, 0x2e, 0x28, 0x48, 0xbc, 0x03, 0xac, 0x25, 0xa8, 0xe0, 0x38, 0x50, 0xd2, 0x39, 0x29,
0x2d, 0xe3, 0x38, 0x90, 0xa2, 0x16, 0x2c, 0xea, 0xb7, 0x4c, 0xc2, 0x64, 0x08, 0x65, 0xd3, 0x31,
0xd7, 0x9d, 0x23, 0x1e, 0x90, 0xdb, 0xac, 0xbb, 0xab, 0x35, 0xdd, 0x05, 0xf5, 0x9c, 0xd1, 0x4b,
0xeb, 0x3d, 0xa8, 0x89, 0x5d, 0x72, 0x43, 0xe5, 0x13, 0x1b, 0x32, 0x71, 0x1c, 0x64, 0x0b, 0xe7,
0x0b, 0x03, 0x4e, 0x3f, 0x43, 0xe1, 0x0c, 0x79, 0x74, 0x17, 0x2a, 0x6d, 0xdc, 0x15, 0x26, 0xb2,
0x17, 0xda, 0xda, 0x51, 0x0f, 0xfe, 0x23, 0x02, 0xe6, 0xe6, 0x06, 0x9c, 0x87, 0x79, 0x58, 0x65,
0xfd, 0x89, 0x97, 0xae, 0x68, 0x2a, 0xc1, 0x4b, 0x28, 0x44, 0xe7, 0x13, 0x43, 0xbc, 0x42, 0x03,
0xfc, 0x54, 0x6e, 0xfd, 0x4c, 0x62, 0x1a, 0xb3, 0x24, 0xa6, 0x75, 0x1d, 0xce, 0xc6, 0x69, 0xe4,
0x51, 0x1c, 0x22, 0x8e, 0x03, 0x4f, 0xef, 0xc6, 0xf4, 0xee, 0x56, 0x9c, 0x46, 0xae, 0x12, 0x69,
0x37, 0x99, 0xf3, 0xb9, 0x01, 0x70, 0x47, 0x54, 0x8f, 0x3a, 0xc6, 0x64, 0x7b, 0x30, 0x8e, 0xbf,
0x92, 0x15, 0xc6, 0xcb, 0x6f, 0x23, 0x2b, 0x3f, 0x26, 0xe3, 0x51, 0x9c, 0xe6, 0x43, 0x1e, 0x8f,
0xa1, 0xf3, 0xba, 0x42, 0x55, 0x0c, 0xbe, 0x34, 0xa0, 0x36, 0x12, 0x2a, 0x36, 0x4e, 0xa3, 0x31,
0xd9, 0x29, 0xe4, 0xbc, 0x10, 0xd5, 0xe3, 0xb1, 0x91, 0x82, 0x8a, 0x86, 0x05, 0xb5, 0x04, 0x15,
0x49, 0xc9, 0x48, 0x45, 0xc5, 0xba, 0xa2, 0xae, 0xc1, 0x69, 0x8a, 0x7d, 0x1c, 0xf3, 0x70, 0xe0,
0x45, 0x49, 0x40, 0xf6, 0x09, 0x0e, 0x64, 0x5d, 0x55, 0xdc, 0x7a, 0x26, 0xd8, 0xd6, 0xb8, 0xf3,
0xa3, 0x01, 0x8b, 0x1f, 0x88, 0x31, 0xba, 0x93, 0x04, 0x58, 0x9d, 0xec, 0xc5, 0x53, 0xe2, 0x96,
0xf4, 0x45, 0xd3, 0xa3, 0xd2, 0xf5, 0x7f, 0xcf, 0x4f, 0x57, 0xe6, 0x56, 0x98, 0x4e, 0x51, 0x41,
0xb1, 0xba, 0x66, 0x9f, 0x84, 0xe2, 0x61, 0x60, 0x5d, 0x75, 0x39, 0x57, 0x14, 0x7f, 0x6c, 0x80,
0x39, 0x52, 0x98, 0x62, 0x26, 0xe9, 0x01, 0xa6, 0xe6, 0x9e, 0x21, 0x1b, 0xae, 0xa9, 0x31, 0xd9,
0x72, 0xcf, 0xc2, 0x5c, 0xc4, 0xba, 0xf9, 0x35, 0x49, 0x2d, 0xac, 0x8b, 0x50, 0x89, 0x58, 0x57,
0xde, 0x46, 0x74, 0x97, 0xce, 0xd7, 0x22, 0x6c, 0xf9, 0xd8, 0xd3, 0xcd, 0x6a, 0x08, 0xbc, 0xf1,
0x0e, 0x54, 0xf3, 0x3f, 0xe0, 0xac, 0x3a, 0xd4, 0x5a, 0x31, 0xe1, 0x04, 0x85, 0xe4, 0x90, 0xc4,
0xdd, 0xfa, 0x7f, 0x2c, 0x13, 0xca, 0xef, 0x63, 0x14, 0xf2, 0xde, 0xa0, 0x6e, 0x58, 0x35, 0xa8,
0xdc, 0xee, 0xc4, 0x09, 0x8d, 0x50, 0x58, 0x2f, 0x6c, 0x34, 0x3f, 0xda, 0xe8, 0x12, 0xde, 0x4b,
0x3b, 0x82, 0xe1, 0xb5, 0x43, 0x12, 0x86, 0xe4, 0x90, 0x63, 0xbf, 0xb7, 0xa6, 0x28, 0x78, 0x33,
0x20, 0x8c, 0x53, 0xd2, 0x49, 0x39, 0x0e, 0xd6, 0x32, 0x22, 0xd6, 0x24, 0x2f, 0xf9, 0xb2, 0xdf,
0xe9, 0xcc, 0x4b, 0xe4, 0xc6, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0x54, 0x28, 0x8f, 0xa8, 0xa5,
0x14, 0x00, 0x00,
}

220
internal/proxynode/meta_cache.go
View File

@ -6,6 +6,9 @@ import (
"fmt"
"sync"
"github.com/zilliztech/milvus-distributed/internal/log"
"go.uber.org/zap"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
@ -16,6 +19,7 @@ import (
type Cache interface {
GetCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error)
GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error)
GetPartitions(ctx context.Context, collectionName string) (map[string]typeutil.UniqueID, error)
GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error)
RemoveCollection(ctx context.Context, collectionName string)
RemovePartition(ctx context.Context, collectionName string, partitionName string)
@ -52,83 +56,143 @@ func NewMetaCache(client types.MasterService) (*MetaCache, error) {
}, nil
}
func (m *MetaCache) readCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error) {
m.mu.RLock()
defer m.mu.RUnlock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
return 0, fmt.Errorf("can't find collection name:%s", collectionName)
}
return collInfo.collID, nil
}
func (m *MetaCache) readCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
m.mu.RLock()
defer m.mu.RUnlock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
return nil, fmt.Errorf("can't find collection name:%s", collectionName)
}
return collInfo.schema, nil
}
func (m *MetaCache) readPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
m.mu.RLock()
defer m.mu.RUnlock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
return 0, fmt.Errorf("can't find collection name:%s", collectionName)
}
partitionID, ok := collInfo.partInfo[partitionName]
if !ok {
return 0, fmt.Errorf("can't find partition name:%s", partitionName)
}
return partitionID, nil
}
func (m *MetaCache) GetCollectionID(ctx context.Context, collectionName string) (typeutil.UniqueID, error) {
collID, err := m.readCollectionID(ctx, collectionName)
if err == nil {
return collID, nil
}
m.mu.Lock()
defer m.mu.Unlock()
m.mu.RLock()
collInfo, ok := m.collInfo[collectionName]
req := &milvuspb.DescribeCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DescribeCollection,
},
CollectionName: collectionName,
}
coll, err := m.client.DescribeCollection(ctx, req)
if !ok {
m.mu.RUnlock()
coll, err := m.describeCollection(ctx, collectionName)
if err != nil {
return 0, err
}
if coll.Status.ErrorCode != commonpb.ErrorCode_Success {
return 0, errors.New(coll.Status.Reason)
m.mu.Lock()
defer m.mu.Unlock()
m.updateCollection(coll, collectionName)
collInfo = m.collInfo[collectionName]
return collInfo.collID, nil
}
defer m.mu.RUnlock()
return collInfo.collID, nil
}
func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
m.mu.RLock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
m.mu.RUnlock()
coll, err := m.describeCollection(ctx, collectionName)
if err != nil {
return nil, err
}
m.mu.Lock()
defer m.mu.Unlock()
m.updateCollection(coll, collectionName)
collInfo = m.collInfo[collectionName]
return collInfo.schema, nil
}
defer m.mu.RUnlock()
return collInfo.schema, nil
}
func (m *MetaCache) updateCollection(coll *milvuspb.DescribeCollectionResponse, collectionName string) {
_, ok := m.collInfo[collectionName]
if !ok {
m.collInfo[collectionName] = &collectionInfo{}
}
m.collInfo[collectionName].schema = coll.Schema
m.collInfo[collectionName].collID = coll.CollectionID
return m.collInfo[collectionName].collID, nil
}
func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) {
collSchema, err := m.readCollectionSchema(ctx, collectionName)
if err == nil {
return collSchema, nil
func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
_, err := m.GetCollectionID(ctx, collectionName)
if err != nil {
return 0, err
}
m.mu.RLock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
m.mu.RUnlock()
return 0, fmt.Errorf("can't find collection name:%s", collectionName)
}
partitionID, ok := collInfo.partInfo[partitionName]
m.mu.RUnlock()
if !ok {
partitions, err := m.showPartitions(ctx, collectionName)
if err != nil {
return 0, err
}
m.mu.Lock()
defer m.mu.Unlock()
log.Debug("proxynode", zap.Any("GetPartitionID:partitions before update", partitions), zap.Any("collectionName", collectionName))
m.updatePartitions(partitions, collectionName)
log.Debug("proxynode", zap.Any("GetPartitionID:partitions after update", partitions), zap.Any("collectionName", collectionName))
partInfo := m.collInfo[collectionName].partInfo
_, ok := partInfo[partitionName]
if !ok {
return 0, fmt.Errorf("partitionID of partitionName:%s can not be find", partitionName)
}
return partInfo[partitionName], nil
}
return partitionID, nil
}
func (m *MetaCache) GetPartitions(ctx context.Context, collectionName string) (map[string]typeutil.UniqueID, error) {
_, err := m.GetCollectionID(ctx, collectionName)
if err != nil {
return nil, err
}
m.mu.RLock()
collInfo, ok := m.collInfo[collectionName]
if !ok {
m.mu.RUnlock()
return nil, fmt.Errorf("can't find collection name:%s", collectionName)
}
if collInfo.partInfo == nil || len(collInfo.partInfo) == 0 {
m.mu.RUnlock()
partitions, err := m.showPartitions(ctx, collectionName)
if err != nil {
return nil, err
}
m.mu.Lock()
defer m.mu.Unlock()
m.updatePartitions(partitions, collectionName)
ret := make(map[string]typeutil.UniqueID)
partInfo := m.collInfo[collectionName].partInfo
for k, v := range partInfo {
ret[k] = v
}
return ret, nil
}
defer m.mu.RUnlock()
ret := make(map[string]typeutil.UniqueID)
partInfo := m.collInfo[collectionName].partInfo
for k, v := range partInfo {
ret[k] = v
}
return ret, nil
}
func (m *MetaCache) describeCollection(ctx context.Context, collectionName string) (*milvuspb.DescribeCollectionResponse, error) {
req := &milvuspb.DescribeCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DescribeCollection,
@ -142,45 +206,34 @@ func (m *MetaCache) GetCollectionSchema(ctx context.Context, collectionName stri
if coll.Status.ErrorCode != commonpb.ErrorCode_Success {
return nil, errors.New(coll.Status.Reason)
}
_, ok := m.collInfo[collectionName]
if !ok {
m.collInfo[collectionName] = &collectionInfo{}
}
m.collInfo[collectionName].schema = coll.Schema
m.collInfo[collectionName].collID = coll.CollectionID
return m.collInfo[collectionName].schema, nil
return coll, nil
}
func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, partitionName string) (typeutil.UniqueID, error) {
partitionID, err := m.readPartitionID(ctx, collectionName, partitionName)
if err == nil {
return partitionID, nil
}
func (m *MetaCache) showPartitions(ctx context.Context, collectionName string) (*milvuspb.ShowPartitionsResponse, error) {
req := &milvuspb.ShowPartitionsRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_ShowPartitions,
},
CollectionName: collectionName,
}
partitions, err := m.client.ShowPartitions(ctx, req)
if err != nil {
return 0, err
return nil, err
}
if partitions.Status.ErrorCode != commonpb.ErrorCode_Success {
return 0, fmt.Errorf("%s", partitions.Status.Reason)
return nil, fmt.Errorf("%s", partitions.Status.Reason)
}
m.mu.Lock()
defer m.mu.Unlock()
if len(partitions.PartitionIDs) != len(partitions.PartitionNames) {
return 0, fmt.Errorf("partition ids len: %d doesn't equal Partition name len %d",
return nil, fmt.Errorf("partition ids len: %d doesn't equal Partition name len %d",
len(partitions.PartitionIDs), len(partitions.PartitionNames))
}
return partitions, nil
}
func (m *MetaCache) updatePartitions(partitions *milvuspb.ShowPartitionsResponse, collectionName string) {
_, ok := m.collInfo[collectionName]
if !ok {
m.collInfo[collectionName] = &collectionInfo{
@ -198,12 +251,7 @@ func (m *MetaCache) GetPartitionID(ctx context.Context, collectionName string, p
partInfo[partitions.PartitionNames[i]] = partitions.PartitionIDs[i]
}
}
_, ok = partInfo[partitionName]
if !ok {
return 0, fmt.Errorf("partitionID of partitionName:%s can not be find", partitionName)
}
return partInfo[partitionName], nil
m.collInfo[collectionName].partInfo = partInfo
}
func (m *MetaCache) RemoveCollection(ctx context.Context, collectionName string) {

8
internal/proxynode/proxy_node.go
View File

@ -74,7 +74,7 @@ func (node *ProxyNode) Init() error {
// todo wait for proxyservice state changed to Healthy
ctx := context.Background()
err := funcutil.WaitForComponentHealthy(ctx, node.proxyService, "ProxyService", 100, time.Millisecond*200)
err := funcutil.WaitForComponentHealthy(ctx, node.proxyService, "ProxyService", 1000000, time.Millisecond*200)
if err != nil {
return err
}
@ -102,7 +102,7 @@ func (node *ProxyNode) Init() error {
// wait for dataservice state changed to Healthy
if node.dataService != nil {
err := funcutil.WaitForComponentHealthy(ctx, node.dataService, "DataService", 100, time.Millisecond*200)
err := funcutil.WaitForComponentHealthy(ctx, node.dataService, "DataService", 1000000, time.Millisecond*200)
if err != nil {
return err
}
@ -110,7 +110,7 @@ func (node *ProxyNode) Init() error {
// wait for queryService state changed to Healthy
if node.queryService != nil {
err := funcutil.WaitForComponentHealthy(ctx, node.queryService, "QueryService", 100, time.Millisecond*200)
err := funcutil.WaitForComponentHealthy(ctx, node.queryService, "QueryService", 1000000, time.Millisecond*200)
if err != nil {
return err
}
@ -118,7 +118,7 @@ func (node *ProxyNode) Init() error {
// wait for indexservice state changed to Healthy
if node.indexService != nil {
err := funcutil.WaitForComponentHealthy(ctx, node.indexService, "IndexService", 100, time.Millisecond*200)
err := funcutil.WaitForComponentHealthy(ctx, node.indexService, "IndexService", 1000000, time.Millisecond*200)
if err != nil {
return err
}

31
internal/proxynode/task.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"math"
"regexp"
"strconv"
"go.uber.org/zap"
@ -552,13 +553,35 @@ func (st *SearchTask) PreExecute(ctx context.Context) error {
}
st.CollectionID = collectionID
st.PartitionIDs = make([]UniqueID, 0)
for _, partitionName := range st.query.PartitionNames {
partitionID, err := globalMetaCache.GetPartitionID(ctx, collectionName, partitionName)
partitionsMap, err := globalMetaCache.GetPartitions(ctx, collectionName)
if err != nil {
continue
return err
}
st.PartitionIDs = append(st.PartitionIDs, partitionID)
partitionsRecord := make(map[UniqueID]bool)
for _, partitionName := range st.query.PartitionNames {
pattern := fmt.Sprintf("^%s$", partitionName)
re, err := regexp.Compile(pattern)
if err != nil {
return errors.New("invalid partition names")
}
found := false
for name, pID := range partitionsMap {
if re.MatchString(name) {
if _, exist := partitionsRecord[pID]; !exist {
st.PartitionIDs = append(st.PartitionIDs, pID)
partitionsRecord[pID] = true
}
found = true
}
}
if !found {
errMsg := fmt.Sprintf("PartitonName: %s not found", partitionName)
return errors.New(errMsg)
}
}
st.Dsl = st.query.Dsl
st.PlaceholderGroup = st.query.PlaceholderGroup

126
internal/querynode/collection_replica.go
View File

@ -68,21 +68,31 @@ type ReplicaInterface interface {
getSegmentsToLoadBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
getSegmentStatistics() []*internalpb.SegmentStats
// excluded segments
initExcludedSegments(collectionID UniqueID)
removeExcludedSegments(collectionID UniqueID)
addExcludedSegments(collectionID UniqueID, segmentIDs []UniqueID) error
getExcludedSegments(collectionID UniqueID) ([]UniqueID, error)
getEnabledSegmentsBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
getSegmentsBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID)
replaceGrowingSegmentBySealedSegment(segment *Segment) error
getTSafe() tSafer
getTSafe(collectionID UniqueID) tSafer
addTSafe(collectionID UniqueID)
removeTSafe(collectionID UniqueID)
freeAll()
}
type collectionReplica struct {
tSafe tSafer
tSafes map[UniqueID]tSafer // map[collectionID]tSafer
mu sync.RWMutex // guards all
collections map[UniqueID]*Collection
partitions map[UniqueID]*Partition
segments map[UniqueID]*Segment
excludedSegments map[UniqueID][]UniqueID // map[collectionID]segmentIDs
}
//----------------------------------------------------------------------------------------------------- collection
@ -101,7 +111,7 @@ func (colReplica *collectionReplica) addCollection(collectionID UniqueID, schema
defer colReplica.mu.Unlock()
if ok := colReplica.hasCollectionPrivate(collectionID); ok {
return fmt.Errorf("collection has been existed, id %d", collectionID)
return errors.New("collection has been loaded, id %d" + strconv.FormatInt(collectionID, 10))
}
var newCollection = newCollection(collectionID, schema)
@ -143,7 +153,7 @@ func (colReplica *collectionReplica) getCollectionByID(collectionID UniqueID) (*
func (colReplica *collectionReplica) getCollectionByIDPrivate(collectionID UniqueID) (*Collection, error) {
collection, ok := colReplica.collections[collectionID]
if !ok {
return nil, fmt.Errorf("cannot find collection, id = %d", collectionID)
return nil, errors.New("collection hasn't been loaded or has been released, collection id = %d" + strconv.FormatInt(collectionID, 10))
}
return collection, nil
@ -195,7 +205,7 @@ func (colReplica *collectionReplica) getVecFieldIDsByCollectionID(collectionID U
}
if len(vecFields) <= 0 {
return nil, fmt.Errorf("no vector field in collection %d", collectionID)
return nil, errors.New("no vector field in collection %d" + strconv.FormatInt(collectionID, 10))
}
return vecFields, nil
@ -228,7 +238,7 @@ func (colReplica *collectionReplica) getFieldsByCollectionIDPrivate(collectionID
}
if len(collection.Schema().Fields) <= 0 {
return nil, fmt.Errorf("no field in collection %d", collectionID)
return nil, errors.New("no field in collection %d" + strconv.FormatInt(collectionID, 10))
}
return collection.Schema().Fields, nil
@ -291,7 +301,7 @@ func (colReplica *collectionReplica) getPartitionByID(partitionID UniqueID) (*Pa
func (colReplica *collectionReplica) getPartitionByIDPrivate(partitionID UniqueID) (*Partition, error) {
partition, ok := colReplica.partitions[partitionID]
if !ok {
return nil, fmt.Errorf("cannot find partition, id = %d", partitionID)
return nil, errors.New("partition hasn't been loaded or has been released, partition id = %d" + strconv.FormatInt(partitionID, 10))
}
return partition, nil
@ -426,7 +436,7 @@ func (colReplica *collectionReplica) getSegmentByID(segmentID UniqueID) (*Segmen
func (colReplica *collectionReplica) getSegmentByIDPrivate(segmentID UniqueID) (*Segment, error) {
segment, ok := colReplica.segments[segmentID]
if !ok {
return nil, errors.New("cannot find segment, id = " + strconv.FormatInt(segmentID, 10))
return nil, errors.New("cannot find segment in query node, id = " + strconv.FormatInt(segmentID, 10))
}
return segment, nil
@ -529,7 +539,7 @@ func (colReplica *collectionReplica) getSegmentsBySegmentType(segType segmentTyp
func (colReplica *collectionReplica) replaceGrowingSegmentBySealedSegment(segment *Segment) error {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
if segment.segmentType != segmentTypeSealed && segment.segmentType != segTypeIndexing {
if segment.segmentType != segmentTypeSealed && segment.segmentType != segmentTypeIndexing {
return errors.New("unexpected segment type")
}
targetSegment, err := colReplica.getSegmentByIDPrivate(segment.ID())
@ -573,9 +583,66 @@ func (colReplica *collectionReplica) setSegmentEnableLoadBinLog(segmentID Unique
return nil
}
func (colReplica *collectionReplica) initExcludedSegments(collectionID UniqueID) {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
colReplica.excludedSegments[collectionID] = make([]UniqueID, 0)
}
func (colReplica *collectionReplica) removeExcludedSegments(collectionID UniqueID) {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
delete(colReplica.excludedSegments, collectionID)
}
func (colReplica *collectionReplica) addExcludedSegments(collectionID UniqueID, segmentIDs []UniqueID) error {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
if _, ok := colReplica.excludedSegments[collectionID]; !ok {
return errors.New("addExcludedSegments failed, cannot found collection, id =" + fmt.Sprintln(collectionID))
}
colReplica.excludedSegments[collectionID] = append(colReplica.excludedSegments[collectionID], segmentIDs...)
return nil
}
func (colReplica *collectionReplica) getExcludedSegments(collectionID UniqueID) ([]UniqueID, error) {
colReplica.mu.RLock()
defer colReplica.mu.RUnlock()
if _, ok := colReplica.excludedSegments[collectionID]; !ok {
return nil, errors.New("getExcludedSegments failed, cannot found collection, id =" + fmt.Sprintln(collectionID))
}
return colReplica.excludedSegments[collectionID], nil
}
//-----------------------------------------------------------------------------------------------------
func (colReplica *collectionReplica) getTSafe() tSafer {
return colReplica.tSafe
func (colReplica *collectionReplica) getTSafe(collectionID UniqueID) tSafer {
colReplica.mu.RLock()
defer colReplica.mu.RUnlock()
return colReplica.getTSafePrivate(collectionID)
}
func (colReplica *collectionReplica) getTSafePrivate(collectionID UniqueID) tSafer {
return colReplica.tSafes[collectionID]
}
func (colReplica *collectionReplica) addTSafe(collectionID UniqueID) {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
colReplica.tSafes[collectionID] = newTSafe()
}
func (colReplica *collectionReplica) removeTSafe(collectionID UniqueID) {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
ts := colReplica.getTSafePrivate(collectionID)
ts.close()
delete(colReplica.tSafes, collectionID)
}
func (colReplica *collectionReplica) freeAll() {
@ -591,24 +658,6 @@ func (colReplica *collectionReplica) freeAll() {
colReplica.segments = make(map[UniqueID]*Segment)
}
func newCollectionReplica() ReplicaInterface {
collections := make(map[int64]*Collection)
partitions := make(map[int64]*Partition)
segments := make(map[int64]*Segment)
tSafe := newTSafe()
var replica ReplicaInterface = &collectionReplica{
collections: collections,
partitions: partitions,
segments: segments,
tSafe: tSafe,
}
return replica
}
func (colReplica *collectionReplica) getSegmentsToLoadBySegmentType(segType segmentType) ([]UniqueID, []UniqueID, []UniqueID) {
colReplica.mu.RLock()
defer colReplica.mu.RUnlock()
@ -634,3 +683,22 @@ func (colReplica *collectionReplica) getSegmentsToLoadBySegmentType(segType segm
return targetCollectionIDs, targetPartitionIDs, targetSegmentIDs
}
func newCollectionReplica() ReplicaInterface {
collections := make(map[UniqueID]*Collection)
partitions := make(map[UniqueID]*Partition)
segments := make(map[UniqueID]*Segment)
excludedSegments := make(map[UniqueID][]UniqueID)
var replica ReplicaInterface = &collectionReplica{
collections: collections,
partitions: partitions,
segments: segments,
excludedSegments: excludedSegments,
tSafes: make(map[UniqueID]tSafer),
}
return replica
}

18
internal/querynode/data_sync_service.go
View File

@ -13,6 +13,9 @@ import (
type dataSyncService struct {
ctx context.Context
cancel context.CancelFunc
collectionID UniqueID
fg *flowgraph.TimeTickedFlowGraph
dmStream msgstream.MsgStream
@ -21,9 +24,13 @@ type dataSyncService struct {
replica ReplicaInterface
}
func newDataSyncService(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory) *dataSyncService {
func newDataSyncService(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory, collectionID UniqueID) *dataSyncService {
ctx1, cancel := context.WithCancel(ctx)
service := &dataSyncService{
ctx: ctx,
ctx: ctx1,
cancel: cancel,
collectionID: collectionID,
fg: nil,
replica: replica,
msFactory: factory,
@ -38,6 +45,7 @@ func (dsService *dataSyncService) start() {
}
func (dsService *dataSyncService) close() {
dsService.cancel()
if dsService.fg != nil {
dsService.fg.Close()
}
@ -50,10 +58,10 @@ func (dsService *dataSyncService) initNodes() {
var dmStreamNode node = dsService.newDmInputNode(dsService.ctx)
var filterDmNode node = newFilteredDmNode(dsService.replica)
var filterDmNode node = newFilteredDmNode(dsService.replica, dsService.collectionID)
var insertNode node = newInsertNode(dsService.replica)
var serviceTimeNode node = newServiceTimeNode(dsService.ctx, dsService.replica, dsService.msFactory)
var insertNode node = newInsertNode(dsService.replica, dsService.collectionID)
var serviceTimeNode node = newServiceTimeNode(dsService.ctx, dsService.replica, dsService.msFactory, dsService.collectionID)
dsService.fg.AddNode(dmStreamNode)

8
internal/querynode/data_sync_service_test.go
View File

@ -18,6 +18,8 @@ import (
func TestDataSyncService_Start(t *testing.T) {
ctx := context.Background()
collectionID := UniqueID(0)
node := newQueryNodeMock()
initTestMeta(t, node, 0, 0)
// test data generate
@ -64,7 +66,7 @@ func TestDataSyncService_Start(t *testing.T) {
Timestamp: uint64(i + 1000),
SourceID: 0,
},
CollectionID: UniqueID(0),
CollectionID: collectionID,
PartitionID: defaultPartitionID,
SegmentID: int64(0),
ChannelID: "0",
@ -132,8 +134,8 @@ func TestDataSyncService_Start(t *testing.T) {
assert.NoError(t, err)
// dataSync
node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
go node.dataSyncService.start()
node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
go node.dataSyncServices[collectionID].start()
<-node.queryNodeLoopCtx.Done()
node.Stop()

51
internal/querynode/flow_graph_filter_dm_node.go
View File

@ -2,6 +2,7 @@ package querynode
import (
"context"
"fmt"
"go.uber.org/zap"
@ -12,6 +13,7 @@ import (
type filterDmNode struct {
baseNode
collectionID UniqueID
replica ReplicaInterface
}
@ -33,6 +35,10 @@ func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, cont
// TODO: add error handling
}
if msgStreamMsg == nil {
return []Msg{}, ctx
}
var iMsg = insertMsg{
insertMessages: make([]*msgstream.InsertMsg, 0),
timeRange: TimeRange{
@ -60,14 +66,31 @@ func (fdmNode *filterDmNode) Operate(ctx context.Context, in []Msg) ([]Msg, cont
}
func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg) *msgstream.InsertMsg {
// TODO: open this check
// check if partition dm enable
enableCollection := fdmNode.replica.hasCollection(msg.CollectionID)
enablePartition := fdmNode.replica.hasPartition(msg.PartitionID)
if !enableCollection || !enablePartition {
// check if collection and partition exist
collection := fdmNode.replica.hasCollection(msg.CollectionID)
partition := fdmNode.replica.hasPartition(msg.PartitionID)
if !collection || !partition {
return nil
}
// check if the collection from message is target collection
if msg.CollectionID != fdmNode.collectionID {
return nil
}
// check if the segment is in excluded segments
excludedSegments, err := fdmNode.replica.getExcludedSegments(fdmNode.collectionID)
log.Debug("excluded segments", zap.String("segmentIDs", fmt.Sprintln(excludedSegments)))
if err != nil {
log.Error(err.Error())
return nil
}
for _, id := range excludedSegments {
if msg.SegmentID == id {
return nil
}
}
// TODO: If the last record is drop type, all insert requests are invalid.
//if !records[len(records)-1].createOrDrop {
// return nil
@ -80,27 +103,14 @@ func (fdmNode *filterDmNode) filterInvalidInsertMessage(msg *msgstream.InsertMsg
return nil
}
tmpTimestamps := make([]Timestamp, 0)
tmpRowIDs := make([]int64, 0)
tmpRowData := make([]*commonpb.Blob, 0)
for i, t := range msg.Timestamps {
tmpTimestamps = append(tmpTimestamps, t)
tmpRowIDs = append(tmpRowIDs, msg.RowIDs[i])
tmpRowData = append(tmpRowData, msg.RowData[i])
}
if len(tmpRowIDs) <= 0 {
if len(msg.Timestamps) <= 0 {
return nil
}
msg.Timestamps = tmpTimestamps
msg.RowIDs = tmpRowIDs
msg.RowData = tmpRowData
return msg
}
func newFilteredDmNode(replica ReplicaInterface) *filterDmNode {
func newFilteredDmNode(replica ReplicaInterface, collectionID UniqueID) *filterDmNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism
@ -110,6 +120,7 @@ func newFilteredDmNode(replica ReplicaInterface) *filterDmNode {
return &filterDmNode{
baseNode: baseNode,
collectionID: collectionID,
replica: replica,
}
}

17
internal/querynode/flow_graph_insert_node.go
View File

@ -12,6 +12,7 @@ import (
type insertNode struct {
baseNode
collectionID UniqueID
replica ReplicaInterface
}
@ -48,6 +49,10 @@ func (iNode *insertNode) Operate(ctx context.Context, in []Msg) ([]Msg, context.
insertOffset: make(map[int64]int64),
}
if iMsg == nil {
return []Msg{}, ctx
}
// 1. hash insertMessages to insertData
for _, task := range iMsg.insertMessages {
// check if segment exists, if not, create this segment
@ -119,6 +124,11 @@ func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *syn
return
}
if targetSegment.segmentType != segmentTypeGrowing {
wg.Done()
return
}
ids := insertData.insertIDs[segmentID]
timestamps := insertData.insertTimestamps[segmentID]
records := insertData.insertRecords[segmentID]
@ -132,11 +142,13 @@ func (iNode *insertNode) insert(insertData *InsertData, segmentID int64, wg *syn
return
}
log.Debug("Do insert done", zap.Int("len", len(insertData.insertIDs[segmentID])))
log.Debug("Do insert done", zap.Int("len", len(insertData.insertIDs[segmentID])),
zap.Int64("segmentID", segmentID),
zap.Int64("collectionID", iNode.collectionID))
wg.Done()
}
func newInsertNode(replica ReplicaInterface) *insertNode {
func newInsertNode(replica ReplicaInterface, collectionID UniqueID) *insertNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism
@ -146,6 +158,7 @@ func newInsertNode(replica ReplicaInterface) *insertNode {
return &insertNode{
baseNode: baseNode,
collectionID: collectionID,
replica: replica,
}
}

17
internal/querynode/flow_graph_service_time_node.go
View File

@ -13,6 +13,7 @@ import (
type serviceTimeNode struct {
baseNode
collectionID UniqueID
replica ReplicaInterface
timeTickMsgStream msgstream.MsgStream
}
@ -35,9 +36,18 @@ func (stNode *serviceTimeNode) Operate(ctx context.Context, in []Msg) ([]Msg, co
// TODO: add error handling
}
if serviceTimeMsg == nil {
return []Msg{}, ctx
}
// update service time
stNode.replica.getTSafe().set(serviceTimeMsg.timeRange.timestampMax)
//log.Debug("update tSafe to:", getPhysicalTime(serviceTimeMsg.timeRange.timestampMax))
ts := stNode.replica.getTSafe(stNode.collectionID)
if ts != nil {
ts.set(serviceTimeMsg.timeRange.timestampMax)
log.Debug("update tSafe:",
zap.Int64("tSafe", int64(serviceTimeMsg.timeRange.timestampMax)),
zap.Int64("collectionID", stNode.collectionID))
}
if err := stNode.sendTimeTick(serviceTimeMsg.timeRange.timestampMax); err != nil {
log.Error("Error: send time tick into pulsar channel failed", zap.Error(err))
@ -71,7 +81,7 @@ func (stNode *serviceTimeNode) sendTimeTick(ts Timestamp) error {
return stNode.timeTickMsgStream.Produce(context.TODO(), &msgPack)
}
func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory) *serviceTimeNode {
func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory msgstream.Factory, collectionID UniqueID) *serviceTimeNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism
@ -85,6 +95,7 @@ func newServiceTimeNode(ctx context.Context, replica ReplicaInterface, factory m
return &serviceTimeNode{
baseNode: baseNode,
collectionID: collectionID,
replica: replica,
timeTickMsgStream: timeTimeMsgStream,
}

15
internal/querynode/index_loader.go
View File

@ -111,7 +111,12 @@ func (loader *indexLoader) execute(l *loadIndex) error {
if err != nil {
return err
}
// 3. update segment index stats
// 3. drop vector field data if index loaded successfully
err = loader.dropVectorFieldData(l.segmentID, l.fieldID)
if err != nil {
return err
}
// 4. update segment index stats
err = loader.updateSegmentIndexStats(indexParams, indexName, indexID, l)
if err != nil {
return err
@ -277,6 +282,14 @@ func (loader *indexLoader) updateSegmentIndex(indexParams indexParam, bytesIndex
return segment.updateSegmentIndex(loadIndexInfo)
}
func (loader *indexLoader) dropVectorFieldData(segmentID UniqueID, vecFieldID int64) error {
segment, err := loader.replica.getSegmentByID(segmentID)
if err != nil {
return err
}
return segment.dropFieldData(vecFieldID)
}
func (loader *indexLoader) sendQueryNodeStats() error {
resultFieldsStats := make([]*internalpb.FieldStats, 0)
for fieldStatsKey, indexStats := range loader.fieldIndexes {

13
internal/querynode/load_service.go
View File

@ -2,19 +2,16 @@ package querynode
import (
"context"
"errors"
"fmt"
"sync"
"time"
"github.com/zilliztech/milvus-distributed/internal/types"
"errors"
"go.uber.org/zap"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/types"
)
const loadingCheckInterval = 3
@ -76,7 +73,7 @@ func (s *loadService) loadSegmentActively(wg *sync.WaitGroup) {
}
// load segment passively
func (s *loadService) loadSegment(collectionID UniqueID, partitionID UniqueID, segmentIDs []UniqueID, fieldIDs []int64) error {
func (s *loadService) loadSegmentPassively(collectionID UniqueID, partitionID UniqueID, segmentIDs []UniqueID, fieldIDs []int64) error {
// TODO: interim solution
if len(fieldIDs) == 0 {
var err error
@ -168,10 +165,10 @@ func (s *loadService) loadSegmentInternal(collectionID UniqueID, partitionID Uni
return nil
}
func newLoadService(ctx context.Context, masterService types.MasterService, dataService types.DataService, indexService types.IndexService, replica ReplicaInterface, dmStream msgstream.MsgStream) *loadService {
func newLoadService(ctx context.Context, masterService types.MasterService, dataService types.DataService, indexService types.IndexService, replica ReplicaInterface) *loadService {
ctx1, cancel := context.WithCancel(ctx)
segLoader := newSegmentLoader(ctx1, masterService, indexService, dataService, replica, dmStream)
segLoader := newSegmentLoader(ctx1, masterService, indexService, dataService, replica)
return &loadService{
ctx: ctx1,

2
internal/querynode/load_service_test.go
View File

@ -1129,7 +1129,7 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
defer node.Stop()
ctx := node.queryNodeLoopCtx
node.loadService = newLoadService(ctx, nil, nil, nil, node.replica, nil)
node.loadService = newLoadService(ctx, nil, nil, nil, node.replica)
initTestMeta(t, node, collectionID, 0)

2
internal/querynode/meta_service.go
View File

@ -46,7 +46,7 @@ func newMetaService(ctx context.Context, replica ReplicaInterface) *metaService
}
return nil
}
err = retry.Retry(200, time.Millisecond*200, connectEtcdFn)
err = retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
panic(err)
}

137
internal/querynode/query_node.go
View File

@ -14,17 +14,14 @@ import "C"
import (
"context"
"errors"
"fmt"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
"math/rand"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/zilliztech/milvus-distributed/internal/types"
"errors"
"go.uber.org/zap"
"github.com/zilliztech/milvus-distributed/internal/log"
@ -33,7 +30,9 @@ import (
"github.com/zilliztech/milvus-distributed/internal/msgstream/rmqms"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
queryPb "github.com/zilliztech/milvus-distributed/internal/proto/querypb"
"github.com/zilliztech/milvus-distributed/internal/types"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
@ -47,7 +46,7 @@ type QueryNode struct {
replica ReplicaInterface
// internal services
dataSyncService *dataSyncService
dataSyncServices map[UniqueID]*dataSyncService
metaService *metaService
searchService *searchService
loadService *loadService
@ -70,7 +69,7 @@ func NewQueryNode(ctx context.Context, queryNodeID UniqueID, factory msgstream.F
queryNodeLoopCancel: cancel,
QueryNodeID: queryNodeID,
dataSyncService: nil,
dataSyncServices: make(map[UniqueID]*dataSyncService),
metaService: nil,
searchService: nil,
statsService: nil,
@ -89,7 +88,7 @@ func NewQueryNodeWithoutID(ctx context.Context, factory msgstream.Factory) *Quer
queryNodeLoopCtx: ctx1,
queryNodeLoopCancel: cancel,
dataSyncService: nil,
dataSyncServices: make(map[UniqueID]*dataSyncService),
metaService: nil,
searchService: nil,
statsService: nil,
@ -167,15 +166,13 @@ func (node *QueryNode) Start() error {
}
// init services and manager
node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, node.msFactory)
node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, node.msFactory)
//node.metaService = newMetaService(node.queryNodeLoopCtx, node.replica)
node.loadService = newLoadService(node.queryNodeLoopCtx, node.masterService, node.dataService, node.indexService, node.replica, node.dataSyncService.dmStream)
node.loadService = newLoadService(node.queryNodeLoopCtx, node.masterService, node.dataService, node.indexService, node.replica)
node.statsService = newStatsService(node.queryNodeLoopCtx, node.replica, node.loadService.segLoader.indexLoader.fieldStatsChan, node.msFactory)
// start services
go node.dataSyncService.start()
go node.searchService.start()
//go node.metaService.start()
go node.loadService.start()
@ -192,8 +189,10 @@ func (node *QueryNode) Stop() error {
node.replica.freeAll()
// close services
if node.dataSyncService != nil {
node.dataSyncService.close()
for _, dsService := range node.dataSyncServices {
if dsService != nil {
dsService.close()
}
}
if node.searchService != nil {
node.searchService.close()
@ -366,17 +365,20 @@ func (node *QueryNode) RemoveQueryChannel(ctx context.Context, in *queryPb.Remov
}
func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmChannelsRequest) (*commonpb.Status, error) {
if node.dataSyncService == nil || node.dataSyncService.dmStream == nil {
errMsg := "null data sync service or null data manipulation stream"
log.Debug("starting WatchDmChannels ...", zap.String("ChannelIDs", fmt.Sprintln(in.ChannelIDs)))
collectionID := in.CollectionID
service, ok := node.dataSyncServices[collectionID]
if !ok || service.dmStream == nil {
errMsg := "null data sync service or null data manipulation stream, collectionID = " + fmt.Sprintln(collectionID)
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: errMsg,
}
log.Error(errMsg)
return status, errors.New(errMsg)
}
switch t := node.dataSyncService.dmStream.(type) {
switch t := service.dmStream.(type) {
case *pulsarms.PulsarTtMsgStream:
case *rmqms.RmqTtMsgStream:
default:
@ -386,19 +388,61 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: errMsg,
}
log.Error(errMsg)
return status, errors.New(errMsg)
}
getUniqueSubName := func() string {
prefixName := Params.MsgChannelSubName
return prefixName + "-" + strconv.FormatInt(collectionID, 10)
}
// add request channel
consumeChannels := in.ChannelIDs
consumeSubName := Params.MsgChannelSubName
node.dataSyncService.dmStream.AsConsumer(consumeChannels, consumeSubName)
toSeekInfo := make([]*internalpb.MsgPosition, 0)
toDirSubChannels := make([]string, 0)
consumeSubName := getUniqueSubName()
for _, info := range in.Infos {
if len(info.Pos.MsgID) == 0 {
toDirSubChannels = append(toDirSubChannels, info.ChannelID)
continue
}
info.Pos.MsgGroup = consumeSubName
toSeekInfo = append(toSeekInfo, info.Pos)
log.Debug("prevent inserting segments", zap.String("segmentIDs", fmt.Sprintln(info.ExcludedSegments)))
err := node.replica.addExcludedSegments(collectionID, info.ExcludedSegments)
if err != nil {
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
log.Error(err.Error())
return status, err
}
}
service.dmStream.AsConsumer(toDirSubChannels, consumeSubName)
for _, pos := range toSeekInfo {
err := service.dmStream.Seek(pos)
if err != nil {
errMsg := "msgStream seek error :" + err.Error()
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: errMsg,
}
log.Error(errMsg)
return status, errors.New(errMsg)
}
}
log.Debug("querynode AsConsumer: " + strings.Join(consumeChannels, ", ") + " : " + consumeSubName)
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}
log.Debug("WatchDmChannels done", zap.String("ChannelIDs", fmt.Sprintln(in.ChannelIDs)))
return status, nil
}
@ -418,12 +462,18 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment
hasCollection := node.replica.hasCollection(collectionID)
hasPartition := node.replica.hasPartition(partitionID)
if !hasCollection {
// loading init
err := node.replica.addCollection(collectionID, schema)
if err != nil {
status.ErrorCode = commonpb.ErrorCode_UnexpectedError
status.Reason = err.Error()
return status, err
}
node.replica.initExcludedSegments(collectionID)
node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, node.msFactory, collectionID)
go node.dataSyncServices[collectionID].start()
node.replica.addTSafe(collectionID)
node.searchService.register(collectionID)
}
if !hasPartition {
err := node.replica.addPartition(collectionID, partitionID)
@ -444,48 +494,28 @@ func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegment
return status, nil
}
if len(in.SegmentIDs) != len(in.SegmentStates) {
err := errors.New("len(segmentIDs) should equal to len(segmentStates)")
status.ErrorCode = commonpb.ErrorCode_UnexpectedError
status.Reason = err.Error()
return status, err
}
// segments are ordered before LoadSegments calling
//var position *internalpb.MsgPosition = nil
for i, state := range in.SegmentStates {
//thisPosition := state.StartPosition
if state.State <= commonpb.SegmentState_Growing {
//if position == nil {
// position = &internalpb2.MsgPosition{
// ChannelName: thisPosition.ChannelName,
// }
//}
segmentIDs = segmentIDs[:i]
break
}
//position = state.StartPosition
}
//err = node.dataSyncService.seekSegment(position)
//if err != nil {
// status := &commonpb.Status{
// ErrorCode: commonpb.ErrorCode_UnexpectedError,
// Reason: err.Error(),
// }
// return status, err
//}
err = node.loadService.loadSegment(collectionID, partitionID, segmentIDs, fieldIDs)
err = node.loadService.loadSegmentPassively(collectionID, partitionID, segmentIDs, fieldIDs)
if err != nil {
status.ErrorCode = commonpb.ErrorCode_UnexpectedError
status.Reason = err.Error()
return status, err
}
log.Debug("LoadSegments done", zap.String("segmentIDs", fmt.Sprintln(in.SegmentIDs)))
return status, nil
}
func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.ReleaseCollectionRequest) (*commonpb.Status, error) {
if _, ok := node.dataSyncServices[in.CollectionID]; ok {
node.dataSyncServices[in.CollectionID].close()
delete(node.dataSyncServices, in.CollectionID)
node.searchService.tSafeMutex.Lock()
delete(node.searchService.tSafeWatcher, in.CollectionID)
node.searchService.tSafeMutex.Unlock()
node.replica.removeTSafe(in.CollectionID)
node.replica.removeExcludedSegments(in.CollectionID)
}
err := node.replica.removeCollection(in.CollectionID)
if err != nil {
status := &commonpb.Status{
@ -495,6 +525,7 @@ func (node *QueryNode) ReleaseCollection(ctx context.Context, in *queryPb.Releas
return status, err
}
log.Debug("ReleaseCollection done", zap.Int64("collectionID", in.CollectionID))
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil

203
internal/querynode/search_service.go
View File

@ -4,12 +4,12 @@ import "C"
import (
"context"
"errors"
"fmt"
"go.uber.org/zap"
"strconv"
"strings"
"sync"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
@ -24,13 +24,14 @@ type searchService struct {
cancel context.CancelFunc
replica ReplicaInterface
tSafeWatcher *tSafeWatcher
tSafeMutex *sync.Mutex
tSafeWatcher map[UniqueID]*tSafeWatcher
serviceableTimeMutex sync.Mutex // guards serviceableTime
serviceableTime Timestamp
serviceableTime map[UniqueID]Timestamp
msgBuffer chan msgstream.TsMsg
unsolvedMsg []msgstream.TsMsg
msgBuffer chan *msgstream.SearchMsg
unsolvedMsg []*msgstream.SearchMsg
searchMsgStream msgstream.MsgStream
searchResultMsgStream msgstream.MsgStream
queryNodeID UniqueID
@ -54,17 +55,18 @@ func newSearchService(ctx context.Context, replica ReplicaInterface, factory msg
log.Debug("querynode AsProducer: " + strings.Join(producerChannels, ", "))
searchServiceCtx, searchServiceCancel := context.WithCancel(ctx)
msgBuffer := make(chan msgstream.TsMsg, receiveBufSize)
unsolvedMsg := make([]msgstream.TsMsg, 0)
msgBuffer := make(chan *msgstream.SearchMsg, receiveBufSize)
unsolvedMsg := make([]*msgstream.SearchMsg, 0)
return &searchService{
ctx: searchServiceCtx,
cancel: searchServiceCancel,
serviceableTime: Timestamp(0),
serviceableTime: make(map[UniqueID]Timestamp),
msgBuffer: msgBuffer,
unsolvedMsg: unsolvedMsg,
replica: replica,
tSafeWatcher: newTSafeWatcher(),
tSafeMutex: &sync.Mutex{},
tSafeWatcher: make(map[UniqueID]*tSafeWatcher),
searchMsgStream: searchStream,
searchResultMsgStream: searchResultStream,
@ -75,7 +77,6 @@ func newSearchService(ctx context.Context, replica ReplicaInterface, factory msg
func (ss *searchService) start() {
ss.searchMsgStream.Start()
ss.searchResultMsgStream.Start()
ss.register()
ss.wait.Add(2)
go ss.receiveSearchMsg()
go ss.doUnsolvedMsgSearch()
@ -92,32 +93,63 @@ func (ss *searchService) close() {
ss.cancel()
}
func (ss *searchService) register() {
tSafe := ss.replica.getTSafe()
tSafe.registerTSafeWatcher(ss.tSafeWatcher)
func (ss *searchService) register(collectionID UniqueID) {
tSafe := ss.replica.getTSafe(collectionID)
ss.tSafeMutex.Lock()
ss.tSafeWatcher[collectionID] = newTSafeWatcher()
ss.tSafeMutex.Unlock()
tSafe.registerTSafeWatcher(ss.tSafeWatcher[collectionID])
}
func (ss *searchService) waitNewTSafe() Timestamp {
func (ss *searchService) waitNewTSafe(collectionID UniqueID) (Timestamp, error) {
// block until dataSyncService updating tSafe
ss.tSafeWatcher.hasUpdate()
timestamp := ss.replica.getTSafe().get()
return timestamp
ss.tSafeWatcher[collectionID].hasUpdate()
ts := ss.replica.getTSafe(collectionID)
if ts != nil {
return ts.get(), nil
}
return 0, errors.New("tSafe closed, collectionID =" + fmt.Sprintln(collectionID))
}
func (ss *searchService) getServiceableTime() Timestamp {
func (ss *searchService) getServiceableTime(collectionID UniqueID) Timestamp {
ss.serviceableTimeMutex.Lock()
defer ss.serviceableTimeMutex.Unlock()
return ss.serviceableTime
//t, ok := ss.serviceableTime[collectionID]
//if !ok {
// return 0, errors.New("cannot found")
//}
return ss.serviceableTime[collectionID]
}
func (ss *searchService) setServiceableTime(t Timestamp) {
func (ss *searchService) setServiceableTime(collectionID UniqueID, t Timestamp) {
ss.serviceableTimeMutex.Lock()
// hard code gracefultime to 1 second
// TODO: use config to set gracefultime
ss.serviceableTime = t + 1000*1000*1000
ss.serviceableTime[collectionID] = t + 1000*1000*1000
ss.serviceableTimeMutex.Unlock()
}
func (ss *searchService) collectionCheck(collectionID UniqueID) error {
// check if collection exists
if _, ok := ss.tSafeWatcher[collectionID]; !ok {
err := errors.New("no collection found, collectionID = " + strconv.FormatInt(collectionID, 10))
log.Error(err.Error())
return err
}
return nil
}
func (ss *searchService) emptySearch(searchMsg *msgstream.SearchMsg) {
err := ss.search(searchMsg)
if err != nil {
log.Error(err.Error())
err2 := ss.publishFailedSearchResult(searchMsg, err.Error())
if err2 != nil {
log.Error("publish FailedSearchResult failed", zap.Error(err2))
}
}
}
func (ss *searchService) receiveSearchMsg() {
defer ss.wait.Done()
for {
@ -129,26 +161,34 @@ func (ss *searchService) receiveSearchMsg() {
if msgPack == nil || len(msgPack.Msgs) <= 0 {
continue
}
searchMsg := make([]msgstream.TsMsg, 0)
serverTime := ss.getServiceableTime()
for i, msg := range msgPack.Msgs {
if msg.BeginTs() > serverTime {
ss.msgBuffer <- msg
searchNum := 0
for _, msg := range msgPack.Msgs {
sm, ok := msg.(*msgstream.SearchMsg)
if !ok {
continue
}
searchMsg = append(searchMsg, msgPack.Msgs[i])
err := ss.collectionCheck(sm.CollectionID)
if err != nil {
ss.emptySearch(sm)
searchNum++
continue
}
for _, msg := range searchMsg {
err := ss.search(msg)
serviceTime := ss.getServiceableTime(sm.CollectionID)
if msg.BeginTs() > serviceTime {
ss.msgBuffer <- sm
continue
}
err = ss.search(sm)
if err != nil {
log.Error(err.Error())
err2 := ss.publishFailedSearchResult(msg, err.Error())
err2 := ss.publishFailedSearchResult(sm, err.Error())
if err2 != nil {
log.Error("publish FailedSearchResult failed", zap.Error(err2))
}
}
searchNum++
}
log.Debug("ReceiveSearchMsg, do search done", zap.Int("num of searchMsg", len(searchMsg)))
log.Debug("ReceiveSearchMsg, do search done", zap.Int("num of searchMsg", searchNum))
}
}
}
@ -160,18 +200,36 @@ func (ss *searchService) doUnsolvedMsgSearch() {
case <-ss.ctx.Done():
return
default:
serviceTime := ss.waitNewTSafe()
ss.setServiceableTime(serviceTime)
searchMsg := make([]msgstream.TsMsg, 0)
tempMsg := make([]msgstream.TsMsg, 0)
searchMsg := make([]*msgstream.SearchMsg, 0)
tempMsg := make([]*msgstream.SearchMsg, 0)
tempMsg = append(tempMsg, ss.unsolvedMsg...)
ss.unsolvedMsg = ss.unsolvedMsg[:0]
for _, msg := range tempMsg {
if msg.EndTs() <= serviceTime {
searchMsg = append(searchMsg, msg)
serviceTimeTmpTable := make(map[UniqueID]Timestamp)
searchNum := 0
for _, sm := range tempMsg {
err := ss.collectionCheck(sm.CollectionID)
if err != nil {
ss.emptySearch(sm)
searchNum++
continue
}
ss.unsolvedMsg = append(ss.unsolvedMsg, msg)
_, ok := serviceTimeTmpTable[sm.CollectionID]
if !ok {
serviceTime, err := ss.waitNewTSafe(sm.CollectionID)
if err != nil {
// TODO: emptySearch or continue, note: collection has been released
continue
}
ss.setServiceableTime(sm.CollectionID, serviceTime)
serviceTimeTmpTable[sm.CollectionID] = serviceTime
}
if sm.EndTs() <= serviceTimeTmpTable[sm.CollectionID] {
searchMsg = append(searchMsg, sm)
continue
}
ss.unsolvedMsg = append(ss.unsolvedMsg, sm)
}
for {
@ -179,40 +237,52 @@ func (ss *searchService) doUnsolvedMsgSearch() {
if msgBufferLength <= 0 {
break
}
msg := <-ss.msgBuffer
if msg.EndTs() <= serviceTime {
searchMsg = append(searchMsg, msg)
sm := <-ss.msgBuffer
err := ss.collectionCheck(sm.CollectionID)
if err != nil {
ss.emptySearch(sm)
searchNum++
continue
}
ss.unsolvedMsg = append(ss.unsolvedMsg, msg)
_, ok := serviceTimeTmpTable[sm.CollectionID]
if !ok {
serviceTime, err := ss.waitNewTSafe(sm.CollectionID)
if err != nil {
// TODO: emptySearch or continue, note: collection has been released
continue
}
ss.setServiceableTime(sm.CollectionID, serviceTime)
serviceTimeTmpTable[sm.CollectionID] = serviceTime
}
if sm.EndTs() <= serviceTimeTmpTable[sm.CollectionID] {
searchMsg = append(searchMsg, sm)
continue
}
ss.unsolvedMsg = append(ss.unsolvedMsg, sm)
}
if len(searchMsg) <= 0 {
continue
}
for _, msg := range searchMsg {
err := ss.search(msg)
for _, sm := range searchMsg {
err := ss.search(sm)
if err != nil {
log.Error(err.Error())
err2 := ss.publishFailedSearchResult(msg, err.Error())
err2 := ss.publishFailedSearchResult(sm, err.Error())
if err2 != nil {
log.Error("publish FailedSearchResult failed", zap.Error(err2))
}
}
searchNum++
}
log.Debug("doUnsolvedMsgSearch, do search done", zap.Int("num of searchMsg", len(searchMsg)))
log.Debug("doUnsolvedMsgSearch, do search done", zap.Int("num of searchMsg", searchNum))
}
}
}
// TODO:: cache map[dsl]plan
// TODO: reBatched search requests
func (ss *searchService) search(msg msgstream.TsMsg) error {
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
return errors.New("invalid request type = " + string(msg.Type()))
}
func (ss *searchService) search(searchMsg *msgstream.SearchMsg) error {
searchTimestamp := searchMsg.Base.Timestamp
var queryBlob = searchMsg.Query.Value
query := milvuspb.SearchRequest{}
@ -250,21 +320,17 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
partitionIDsInQuery := searchMsg.PartitionIDs
if len(partitionIDsInQuery) == 0 {
if len(partitionIDsInCol) == 0 {
return errors.New("can't find any partition in this collection on query node")
return errors.New("none of this collection's partition has been loaded")
}
searchPartitionIDs = partitionIDsInCol
} else {
findPartition := false
for _, id := range partitionIDsInQuery {
_, err := ss.replica.getPartitionByID(id)
if err == nil {
searchPartitionIDs = append(searchPartitionIDs, id)
findPartition = true
_, err2 := ss.replica.getPartitionByID(id)
if err2 != nil {
return err2
}
}
if !findPartition {
return errors.New("partition to be searched not exist in query node")
}
searchPartitionIDs = partitionIDsInQuery
}
for _, partitionID := range searchPartitionIDs {
@ -380,14 +446,15 @@ func (ss *searchService) search(msg msgstream.TsMsg) error {
}
// For debugging, please don't delete.
//fmt.Println("==================== search result ======================")
//for i := 0; i < len(hits); i++ {
// testHits := milvuspb.Hits{}
// err := proto.Unmarshal(hits[i], &testHits)
// if err != nil {
// panic(err)
// }
// log.Debug(testHits.IDs)
// log.Debug(testHits.Scores)
// fmt.Println(testHits.IDs)
// fmt.Println(testHits.Scores)
//}
err = ss.publishSearchResult(searchResultMsg)
if err != nil {
@ -412,16 +479,12 @@ func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
return err
}
func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg, errMsg string) error {
func (ss *searchService) publishFailedSearchResult(searchMsg *msgstream.SearchMsg, errMsg string) error {
// span, ctx := opentracing.StartSpanFromContext(msg.GetMsgContext(), "receive search msg")
// defer span.Finish()
// msg.SetMsgContext(ctx)
//log.Debug("Public fail SearchResult!")
msgPack := msgstream.MsgPack{}
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
return errors.New("invalid request type = " + string(msg.Type()))
}
resultChannelInt, _ := strconv.ParseInt(searchMsg.ResultChannelID, 10, 64)
searchResultMsg := &msgstream.SearchResultMsg{

21
internal/querynode/search_service_test.go
View File

@ -21,6 +21,8 @@ import (
func TestSearch_Search(t *testing.T) {
ctx := context.Background()
collectionID := UniqueID(0)
node := newQueryNodeMock()
initTestMeta(t, node, 0, 0)
@ -111,6 +113,8 @@ func TestSearch_Search(t *testing.T) {
node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, msFactory)
go node.searchService.start()
node.replica.addTSafe(collectionID)
node.searchService.register(collectionID)
// start insert
timeRange := TimeRange{
@ -143,7 +147,7 @@ func TestSearch_Search(t *testing.T) {
Timestamp: uint64(10 + 1000),
SourceID: 0,
},
CollectionID: UniqueID(0),
CollectionID: collectionID,
PartitionID: defaultPartitionID,
SegmentID: int64(0),
ChannelID: "0",
@ -209,8 +213,8 @@ func TestSearch_Search(t *testing.T) {
assert.NoError(t, err)
// dataSync
node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
go node.dataSyncService.start()
node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
go node.dataSyncServices[collectionID].start()
time.Sleep(1 * time.Second)
@ -219,6 +223,9 @@ func TestSearch_Search(t *testing.T) {
func TestSearch_SearchMultiSegments(t *testing.T) {
ctx := context.Background()
collectionID := UniqueID(0)
pulsarURL := Params.PulsarAddress
const receiveBufSize = 1024
@ -309,6 +316,8 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
node.searchService = newSearchService(node.queryNodeLoopCtx, node.replica, msFactory)
go node.searchService.start()
node.replica.addTSafe(collectionID)
node.searchService.register(collectionID)
// start insert
timeRange := TimeRange{
@ -345,7 +354,7 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
Timestamp: uint64(i + 1000),
SourceID: 0,
},
CollectionID: UniqueID(0),
CollectionID: collectionID,
PartitionID: defaultPartitionID,
SegmentID: int64(segmentID),
ChannelID: "0",
@ -411,8 +420,8 @@ func TestSearch_SearchMultiSegments(t *testing.T) {
assert.NoError(t, err)
// dataSync
node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory)
go node.dataSyncService.start()
node.dataSyncServices[collectionID] = newDataSyncService(node.queryNodeLoopCtx, node.replica, msFactory, collectionID)
go node.dataSyncServices[collectionID].start()
time.Sleep(1 * time.Second)

115
internal/querynode/segment.go
View File

@ -12,6 +12,7 @@ package querynode
*/
import "C"
import (
"fmt"
"strconv"
"sync"
"unsafe"
@ -31,7 +32,7 @@ const (
segmentTypeInvalid segmentType = iota
segmentTypeGrowing
segmentTypeSealed
segTypeIndexing
segmentTypeIndexing
)
type indexParam = map[string]string
@ -268,34 +269,6 @@ func (s *Segment) fillTargetEntry(plan *Plan,
return nil
}
// segment, err := loadService.replica.getSegmentByID(segmentID)
func (s *Segment) updateSegmentIndex(loadIndexInfo *LoadIndexInfo) error {
if s.segmentPtr == nil {
return errors.New("null seg core pointer")
}
var status C.CStatus
if s.segmentType == segmentTypeGrowing {
status = C.UpdateSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
} else if s.segmentType == segmentTypeSealed {
status = C.UpdateSealedSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
} else {
return errors.New("illegal segment type")
}
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("updateSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
s.setType(segTypeIndexing)
return nil
}
func (s *Segment) setIndexParam(fieldID int64, indexParamKv []*commonpb.KeyValuePair) error {
s.paramMutex.Lock()
defer s.paramMutex.Unlock()
@ -461,7 +434,8 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int, data interfa
return errors.New("null seg core pointer")
}
if s.segmentType != segmentTypeSealed {
return errors.New("illegal segment type when loading field data")
errMsg := fmt.Sprintln("segmentLoadFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
return errors.New(errMsg)
}
// data interface check
@ -536,7 +510,86 @@ func (s *Segment) segmentLoadFieldData(fieldID int64, rowCount int, data interfa
return errors.New("LoadFieldData failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
log.Debug("load field done", zap.Int64("fieldID", fieldID), zap.Int("row count", rowCount))
log.Debug("load field done",
zap.Int64("fieldID", fieldID),
zap.Int("row count", rowCount),
zap.Int64("segmentID", s.ID()))
return nil
}
func (s *Segment) dropFieldData(fieldID int64) error {
/*
CStatus
DropFieldData(CSegmentInterface c_segment, int64_t field_id);
*/
if s.segmentPtr == nil {
return errors.New("null seg core pointer")
}
if s.segmentType != segmentTypeIndexing {
errMsg := fmt.Sprintln("dropFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
return errors.New(errMsg)
}
var status = C.DropFieldData(s.segmentPtr, C.long(fieldID))
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("dropFieldData failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
log.Debug("dropFieldData done", zap.Int64("fieldID", fieldID), zap.Int64("segmentID", s.ID()))
return nil
}
func (s *Segment) updateSegmentIndex(loadIndexInfo *LoadIndexInfo) error {
if s.segmentPtr == nil {
return errors.New("null seg core pointer")
}
if s.segmentType != segmentTypeSealed {
errMsg := fmt.Sprintln("updateSegmentIndex failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
return errors.New(errMsg)
}
status := C.UpdateSealedSegmentIndex(s.segmentPtr, loadIndexInfo.cLoadIndexInfo)
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("updateSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
s.setType(segmentTypeIndexing)
log.Debug("updateSegmentIndex done", zap.Int64("segmentID", s.ID()))
return nil
}
func (s *Segment) dropSegmentIndex(fieldID int64) error {
/*
CStatus
DropSealedSegmentIndex(CSegmentInterface c_segment, int64_t field_id);
*/
if s.segmentPtr == nil {
return errors.New("null seg core pointer")
}
if s.segmentType != segmentTypeIndexing {
errMsg := fmt.Sprintln("dropFieldData failed, illegal segment type ", s.segmentType, "segmentID = ", s.ID())
return errors.New(errMsg)
}
var status = C.DropSealedSegmentIndex(s.segmentPtr, C.long(fieldID))
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("dropSegmentIndex failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
log.Debug("dropSegmentIndex done", zap.Int64("fieldID", fieldID), zap.Int64("segmentID", s.ID()))
return nil
}

19
internal/querynode/segment_loader.go
View File

@ -2,31 +2,25 @@ package querynode
import (
"context"
"strconv"
"github.com/zilliztech/milvus-distributed/internal/types"
"errors"
"strconv"
"github.com/zilliztech/milvus-distributed/internal/kv"
minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/types"
)
// segmentLoader is only responsible for loading the field data from binlog
type segmentLoader struct {
replica ReplicaInterface
dmStream msgstream.MsgStream
dataService types.DataService
kv kv.Base // minio kv
iCodec *storage.InsertCodec
indexLoader *indexLoader
}
@ -117,6 +111,8 @@ func (loader *segmentLoader) checkTargetFields(paths []*internalpb.StringList, s
}
func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetFields map[int64]*internalpb.StringList) error {
iCodec := storage.InsertCodec{}
defer iCodec.Close()
for id, p := range targetFields {
if id == timestampFieldID {
// seg core doesn't need timestamp field
@ -136,7 +132,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetField
Value: []byte(binLog),
})
}
_, _, insertData, err := loader.iCodec.Deserialize(blobs)
_, _, insertData, err := iCodec.Deserialize(blobs)
if err != nil {
// TODO: return or continue
return err
@ -193,7 +189,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetField
return nil
}
func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface, dmStream msgstream.MsgStream) *segmentLoader {
func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface) *segmentLoader {
option := &minioKV.Option{
Address: Params.MinioEndPoint,
AccessKeyID: Params.MinioAccessKeyID,
@ -212,12 +208,9 @@ func newSegmentLoader(ctx context.Context, masterService types.MasterService, in
return &segmentLoader{
replica: replica,
dmStream: dmStream,
dataService: dataService,
kv: client,
iCodec: &storage.InsertCodec{},
indexLoader: iLoader,
}

10
internal/querynode/tsafe.go
View File

@ -28,6 +28,7 @@ type tSafer interface {
get() Timestamp
set(t Timestamp)
registerTSafeWatcher(t *tSafeWatcher)
close()
}
type tSafe struct {
@ -64,3 +65,12 @@ func (ts *tSafe) set(t Timestamp) {
watcher.notify()
}
}
func (ts *tSafe) close() {
ts.tSafeMu.Lock()
defer ts.tSafeMu.Unlock()
for _, watcher := range ts.watcherList {
close(watcher.notifyChan)
}
}

43
internal/queryservice/meta_replica.go
View File

@ -1,10 +1,9 @@
package queryservice
import (
"strconv"
"errors"
internalPb "github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/querypb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
)
@ -21,8 +20,9 @@ type Replica interface {
getPartitionStates(dbID UniqueID, collectionID UniqueID, partitionIDs []UniqueID) ([]*querypb.PartitionStates, error)
releaseCollection(dbID UniqueID, collectionID UniqueID) error
releasePartition(dbID UniqueID, collectionID UniqueID, partitionID UniqueID) error
addDmChannels(dbID UniqueID, collectionID UniqueID, channels2NodeID map[string]int64) error
getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error)
addDmChannel(dbID UniqueID, collectionID UniqueID, channel string, watchedStartPos *internalPb.MsgPosition) error
addExcludeSegmentIDs(dbID UniqueID, collectionID UniqueID, excludeSegments []UniqueID) error
//getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error)
}
type segment struct {
@ -38,7 +38,9 @@ type partition struct {
type collection struct {
id UniqueID
partitions map[UniqueID]*partition
dmChannels2Node map[string]int64
dmChannels []string
dmChannels2Pos map[string]*internalPb.MsgPosition
excludeSegmentIds []UniqueID
schema *schemapb.CollectionSchema
}
@ -62,12 +64,16 @@ func (mp *metaReplica) addCollection(dbID UniqueID, collectionID UniqueID, schem
//TODO:: assert dbID = 0 exist
if _, ok := mp.db2collections[dbID]; ok {
partitions := make(map[UniqueID]*partition)
channels := make(map[string]int64)
channels := make([]string, 0)
startPos := make(map[string]*internalPb.MsgPosition)
excludeSegmentIDs := make([]UniqueID, 0)
newCollection := &collection{
id: collectionID,
partitions: partitions,
schema: schema,
dmChannels2Node: channels,
dmChannels: channels,
dmChannels2Pos: startPos,
excludeSegmentIds: excludeSegmentIDs,
}
mp.db2collections[dbID] = append(mp.db2collections[dbID], newCollection)
return nil
@ -216,8 +222,7 @@ func (mp *metaReplica) releaseCollection(dbID UniqueID, collectionID UniqueID) e
}
}
errorStr := "releaseCollection: can't find dbID or collectionID " + strconv.FormatInt(collectionID, 10)
return errors.New(errorStr)
return nil
}
func (mp *metaReplica) releasePartition(dbID UniqueID, collectionID UniqueID, partitionID UniqueID) error {
@ -232,17 +237,15 @@ func (mp *metaReplica) releasePartition(dbID UniqueID, collectionID UniqueID, pa
}
}
errorStr := "releasePartition: can't find dbID or collectionID or partitionID " + strconv.FormatInt(partitionID, 10)
return errors.New(errorStr)
return nil
}
func (mp *metaReplica) addDmChannels(dbID UniqueID, collectionID UniqueID, channels2NodeID map[string]int64) error {
func (mp *metaReplica) addDmChannel(dbID UniqueID, collectionID UniqueID, channel string, watchedStartPos *internalPb.MsgPosition) error {
if collections, ok := mp.db2collections[dbID]; ok {
for _, collection := range collections {
if collectionID == collection.id {
for channel, id := range channels2NodeID {
collection.dmChannels2Node[channel] = id
}
collection.dmChannels = append(collection.dmChannels, channel)
collection.dmChannels2Pos[channel] = watchedStartPos
return nil
}
}
@ -250,16 +253,14 @@ func (mp *metaReplica) addDmChannels(dbID UniqueID, collectionID UniqueID, chann
return errors.New("addDmChannels: can't find dbID or collectionID")
}
func (mp *metaReplica) getAssignedNodeIDByChannelName(dbID UniqueID, collectionID UniqueID, channel string) (int64, error) {
func (mp *metaReplica) addExcludeSegmentIDs(dbID UniqueID, collectionID UniqueID, excludeSegments []UniqueID) error {
if collections, ok := mp.db2collections[dbID]; ok {
for _, collection := range collections {
if collectionID == collection.id {
if id, ok := collection.dmChannels2Node[channel]; ok {
return id, nil
collection.excludeSegmentIds = append(collection.excludeSegmentIds, excludeSegments...)
return nil
}
}
}
}
return 0, errors.New("getAssignedNodeIDByChannelName: can't find dbID or collectionID")
return errors.New("addExcludeSegmentIDs: can't find dbID or collectionID")
}

34
internal/queryservice/querynode.go
View File

@ -11,8 +11,8 @@ import (
type queryNodeInfo struct {
client types.QueryNode
segments []UniqueID
dmChannelNames []string
segments map[UniqueID][]UniqueID
channels2Col map[UniqueID][]string
}
func (qn *queryNodeInfo) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) {
@ -31,8 +31,20 @@ func (qn *queryNodeInfo) WatchDmChannels(ctx context.Context, in *querypb.WatchD
return qn.client.WatchDmChannels(ctx, in)
}
func (qn *queryNodeInfo) AddDmChannels(channels []string) {
qn.dmChannelNames = append(qn.dmChannelNames, channels...)
func (qn *queryNodeInfo) AddDmChannels(channels []string, collectionID UniqueID) {
if _, ok := qn.channels2Col[collectionID]; !ok {
chs := make([]string, 0)
qn.channels2Col[collectionID] = chs
}
qn.channels2Col[collectionID] = append(qn.channels2Col[collectionID], channels...)
}
func (qn *queryNodeInfo) AddSegments(segmentIDs []UniqueID, collectionID UniqueID) {
if _, ok := qn.segments[collectionID]; !ok {
seg := make([]UniqueID, 0)
qn.segments[collectionID] = seg
}
qn.segments[collectionID] = append(qn.segments[collectionID], segmentIDs...)
}
func (qn *queryNodeInfo) AddQueryChannel(ctx context.Context, in *querypb.AddQueryChannelRequest) (*commonpb.Status, error) {
@ -40,7 +52,13 @@ func (qn *queryNodeInfo) AddQueryChannel(ctx context.Context, in *querypb.AddQue
}
func (qn *queryNodeInfo) ReleaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) {
return qn.client.ReleaseCollection(ctx, in)
status, err := qn.client.ReleaseCollection(ctx, in)
if err != nil {
return status, err
}
delete(qn.segments, in.CollectionID)
delete(qn.channels2Col, in.CollectionID)
return status, nil
}
func (qn *queryNodeInfo) ReleasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) {
@ -48,11 +66,11 @@ func (qn *queryNodeInfo) ReleasePartitions(ctx context.Context, in *querypb.Rele
}
func newQueryNodeInfo(client types.QueryNode) *queryNodeInfo {
segments := make([]UniqueID, 0)
dmChannelNames := make([]string, 0)
segments := make(map[UniqueID][]UniqueID)
channels := make(map[UniqueID][]string)
return &queryNodeInfo{
client: client,
segments: segments,
dmChannelNames: dmChannelNames,
channels2Col: channels,
}
}

266
internal/queryservice/queryservice.go
View File

@ -14,17 +14,17 @@ import (
"github.com/opentracing/opentracing-go"
"github.com/uber/jaeger-client-go/config"
nodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/querynode/client"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/types"
"go.uber.org/zap"
nodeclient "github.com/zilliztech/milvus-distributed/internal/distributed/querynode/client"
"github.com/zilliztech/milvus-distributed/internal/log"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/datapb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/milvuspb"
"github.com/zilliztech/milvus-distributed/internal/proto/querypb"
"github.com/zilliztech/milvus-distributed/internal/types"
"github.com/zilliztech/milvus-distributed/internal/util/retry"
)
@ -232,11 +232,6 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
if err != nil {
return fn(err), err
}
err = qs.watchDmChannels(dbID, collectionID)
if err != nil {
return fn(err), err
}
}
// get partitionIDs
@ -297,9 +292,15 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
log.Error("LoadCollectionRequest failed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Error(err))
return status, fmt.Errorf("load partitions: %s", err)
}
err = qs.watchDmChannels(dbID, collectionID)
if err != nil {
log.Error("LoadCollectionRequest failed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Error(err))
return fn(err), err
}
log.Debug("LoadCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID))
return status, nil
}
func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) {
@ -330,7 +331,7 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele
}, err
}
log.Debug("release collection end")
log.Debug("release collection end", zap.Int64("collectionID", collectionID))
//TODO:: queryNode cancel subscribe dmChannels
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
@ -388,16 +389,14 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
return fn(err), err
}
watchNeeded := false
_, err := qs.replica.getCollectionByID(dbID, collectionID)
if err != nil {
err = qs.replica.addCollection(dbID, collectionID, schema)
if err != nil {
return fn(err), err
}
err = qs.watchDmChannels(dbID, collectionID)
if err != nil {
return fn(err), err
}
watchNeeded = true
}
for _, partitionID := range partitionIDs {
@ -446,7 +445,7 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
return fn(err), err
}
for _, state := range resp.States {
log.Error("segment ", zap.String("state.SegmentID", fmt.Sprintln(state.SegmentID)), zap.String("state", fmt.Sprintln(state.StartPosition)))
log.Debug("segment ", zap.String("state.SegmentID", fmt.Sprintln(state.SegmentID)), zap.String("state", fmt.Sprintln(state.StartPosition)))
segmentID := state.SegmentID
segmentStates[segmentID] = state
channelName := state.StartPosition.ChannelName
@ -459,38 +458,79 @@ func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPar
}
}
excludeSegment := make([]UniqueID, 0)
for id, state := range segmentStates {
if state.State > commonpb.SegmentState_Growing {
excludeSegment = append(excludeSegment, id)
}
}
for channel, segmentIDs := range channel2segs {
sort.Slice(segmentIDs, func(i, j int) bool {
return segmentStates[segmentIDs[i]].StartPosition.Timestamp < segmentStates[segmentIDs[j]].StartPosition.Timestamp
})
toLoadSegmentIDs := make([]UniqueID, 0)
var watchedStartPos *internalpb.MsgPosition = nil
var startPosition *internalpb.MsgPosition = nil
for index, id := range segmentIDs {
if segmentStates[id].State <= commonpb.SegmentState_Growing {
if index > 0 {
pos := segmentStates[id].StartPosition
if len(pos.MsgID) == 0 {
watchedStartPos = startPosition
break
}
}
watchedStartPos = segmentStates[id].StartPosition
break
}
toLoadSegmentIDs = append(toLoadSegmentIDs, id)
watchedStartPos = segmentStates[id].EndPosition
startPosition = segmentStates[id].StartPosition
}
if watchedStartPos == nil {
watchedStartPos = &internalpb.MsgPosition{
ChannelName: channel,
}
}
states := make([]*datapb.SegmentStateInfo, 0)
for _, id := range segmentIDs {
states = append(states, segmentStates[id])
}
loadSegmentRequest := &querypb.LoadSegmentsRequest{
CollectionID: collectionID,
PartitionID: partitionID,
SegmentIDs: segmentIDs,
SegmentStates: states,
Schema: schema,
}
nodeID, err := qs.replica.getAssignedNodeIDByChannelName(dbID, collectionID, channel)
err = qs.replica.addDmChannel(dbID, collectionID, channel, watchedStartPos)
if err != nil {
return fn(err), err
}
err = qs.replica.addExcludeSegmentIDs(dbID, collectionID, toLoadSegmentIDs)
if err != nil {
return fn(err), err
}
segment2Node := qs.shuffleSegmentsToQueryNode(toLoadSegmentIDs)
for nodeID, assignedSegmentIDs := range segment2Node {
loadSegmentRequest := &querypb.LoadSegmentsRequest{
CollectionID: collectionID,
PartitionID: partitionID,
SegmentIDs: assignedSegmentIDs,
Schema: schema,
}
queryNode := qs.queryNodes[nodeID]
//TODO:: seek when loadSegment may cause more msgs consumed
//TODO:: all query node should load partition's msg
status, err := queryNode.LoadSegments(ctx, loadSegmentRequest)
if err != nil {
return status, err
}
queryNode.AddSegments(assignedSegmentIDs, collectionID)
}
}
qs.replica.updatePartitionState(dbID, collectionID, partitionID, querypb.PartitionState_InMemory)
}
if watchNeeded {
err = qs.watchDmChannels(dbID, collectionID)
if err != nil {
log.Debug("LoadPartitionRequest completed", zap.Int64("msgID", req.Base.MsgID), zap.Int64s("partitionIDs", partitionIDs), zap.Error(err))
return fn(err), err
}
}
log.Debug("LoadPartitionRequest completed", zap.Int64("msgID", req.Base.MsgID), zap.Int64s("partitionIDs", partitionIDs))
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
@ -529,7 +569,7 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele
}
}
log.Debug("start release partitions end")
log.Debug("start release partitions end", zap.String("partitionIDs", fmt.Sprintln(partitionIDs)))
//TODO:: queryNode cancel subscribe dmChannels
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
@ -683,93 +723,159 @@ func (qs *QueryService) watchDmChannels(dbID UniqueID, collectionID UniqueID) er
}
dmChannels := resp.Values
watchedChannels2NodeID := make(map[string]int64)
unWatchedChannels := make([]string, 0)
channelsWithoutPos := make([]string, 0)
for _, channel := range dmChannels {
findChannel := false
for nodeID, node := range qs.queryNodes {
watchedChannels := node.dmChannelNames
for _, watchedChannel := range watchedChannels {
if channel == watchedChannel {
ChannelsWithPos := collection.dmChannels
for _, ch := range ChannelsWithPos {
if channel == ch {
findChannel = true
watchedChannels2NodeID[channel] = nodeID
break
}
}
}
if !findChannel {
unWatchedChannels = append(unWatchedChannels, channel)
channelsWithoutPos = append(channelsWithoutPos, channel)
}
}
channels2NodeID := qs.shuffleChannelsToQueryNode(unWatchedChannels)
err = qs.replica.addDmChannels(dbID, collection.id, channels2NodeID)
for _, ch := range channelsWithoutPos {
pos := &internalpb.MsgPosition{
ChannelName: ch,
}
err = qs.replica.addDmChannel(dbID, collectionID, ch, pos)
if err != nil {
return err
}
err = qs.replica.addDmChannels(dbID, collection.id, watchedChannels2NodeID)
if err != nil {
return err
}
node2channels := make(map[int64][]string)
for channel, nodeID := range channels2NodeID {
if _, ok := node2channels[nodeID]; ok {
node2channels[nodeID] = append(node2channels[nodeID], channel)
} else {
channels := make([]string, 0)
channels = append(channels, channel)
node2channels[nodeID] = channels
}
}
for nodeID, channels := range node2channels {
channels2NodeID := qs.shuffleChannelsToQueryNode(dmChannels)
for nodeID, channels := range channels2NodeID {
node := qs.queryNodes[nodeID]
watchDmChannelsInfo := make([]*querypb.WatchDmChannelInfo, 0)
for _, ch := range channels {
info := &querypb.WatchDmChannelInfo{
ChannelID: ch,
Pos: collection.dmChannels2Pos[ch],
ExcludedSegments: collection.excludeSegmentIds,
}
watchDmChannelsInfo = append(watchDmChannelsInfo, info)
}
request := &querypb.WatchDmChannelsRequest{
CollectionID: collectionID,
ChannelIDs: channels,
Infos: watchDmChannelsInfo,
}
_, err := node.WatchDmChannels(ctx, request)
if err != nil {
return err
}
node.AddDmChannels(channels, collectionID)
log.Debug("query node ", zap.String("nodeID", strconv.FormatInt(nodeID, 10)), zap.String("watch channels", fmt.Sprintln(channels)))
node.AddDmChannels(channels)
}
return nil
}
func (qs *QueryService) shuffleChannelsToQueryNode(dmChannels []string) map[string]int64 {
maxNumDMChannel := 0
res := make(map[string]int64)
if len(dmChannels) == 0 {
return res
func (qs *QueryService) shuffleChannelsToQueryNode(dmChannels []string) map[int64][]string {
maxNumChannels := 0
for _, node := range qs.queryNodes {
numChannels := 0
for _, chs := range node.channels2Col {
numChannels += len(chs)
}
node2lens := make(map[int64]int)
for id, node := range qs.queryNodes {
node2lens[id] = len(node.dmChannelNames)
if numChannels > maxNumChannels {
maxNumChannels = numChannels
}
}
res := make(map[int64][]string)
offset := 0
loopAll := false
for {
lastOffset := offset
for id, len := range node2lens {
if len >= maxNumDMChannel {
maxNumDMChannel = len
} else {
res[dmChannels[offset]] = id
node2lens[id]++
if !loopAll {
for id, node := range qs.queryNodes {
if len(node.segments) >= maxNumChannels {
continue
}
if _, ok := res[id]; !ok {
res[id] = make([]string, 0)
}
res[id] = append(res[id], dmChannels[offset])
offset++
if offset == len(dmChannels) {
return res
}
}
} else {
for id := range qs.queryNodes {
if _, ok := res[id]; !ok {
res[id] = make([]string, 0)
}
res[id] = append(res[id], dmChannels[offset])
offset++
if offset == len(dmChannels) {
return res
}
}
}
if lastOffset == offset {
for id := range node2lens {
res[dmChannels[offset]] = id
node2lens[id]++
offset++
break
loopAll = true
}
}
}
func (qs *QueryService) shuffleSegmentsToQueryNode(segmentIDs []UniqueID) map[int64][]UniqueID {
maxNumSegments := 0
for _, node := range qs.queryNodes {
numSegments := 0
for _, ids := range node.segments {
numSegments += len(ids)
}
if numSegments > maxNumSegments {
maxNumSegments = numSegments
}
}
res := make(map[int64][]UniqueID)
for nodeID := range qs.queryNodes {
segments := make([]UniqueID, 0)
res[nodeID] = segments
}
if len(segmentIDs) == 0 {
return res
}
offset := 0
loopAll := false
for {
lastOffset := offset
if !loopAll {
for id, node := range qs.queryNodes {
if len(node.segments) >= maxNumSegments {
continue
}
if _, ok := res[id]; !ok {
res[id] = make([]UniqueID, 0)
}
res[id] = append(res[id], segmentIDs[offset])
offset++
if offset == len(segmentIDs) {
return res
}
}
} else {
for id := range qs.queryNodes {
if _, ok := res[id]; !ok {
res[id] = make([]UniqueID, 0)
}
res[id] = append(res[id], segmentIDs[offset])
offset++
if offset == len(segmentIDs) {
return res
}
}
}
if lastOffset == offset {
loopAll = true
}
}
if offset == len(dmChannels) {
break
}
}
return res
}

1
internal/util/flowgraph/flow_graph.go
View File

@ -77,6 +77,7 @@ func (fg *TimeTickedFlowGraph) Close() {
}
(*inStream.inStream).Close()
}
// v.Close()
}
}

6
internal/util/flowgraph/input_node.go
View File

@ -2,11 +2,9 @@ package flowgraph
import (
"context"
"log"
"errors"
"github.com/opentracing/opentracing-go"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/util/trace"
)
@ -40,8 +38,6 @@ func (inNode *InputNode) Operate(ctx context.Context, msgs []Msg) ([]Msg, contex
// TODO: add status
if msgPack == nil {
log.Println("null msg pack")
trace.LogError(sp, errors.New("null msg pack"))
return nil, ctx
}

2
internal/util/flowgraph/node.go
View File

@ -77,7 +77,7 @@ func (nodeCtx *nodeCtx) Start(ctx context.Context, wg *sync.WaitGroup) {
log.Println("nodeCtx.downstream length = ", len(nodeCtx.downstream))
}
if len(res) < downstreamLength {
log.Println("node result length = ", len(res))
// log.Println("node result length = ", len(res))
break
}

1
tests/python_test/collection/test_describe_collection.py
View File

@ -68,6 +68,7 @@ class TestDescribeCollection:
@pytest.mark.tags("0331", "l1")
def test_describe_collection_after_index_created(self, connect, collection, get_simple_index):
connect.create_index(collection, default_float_vec_field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, default_float_vec_field_name)
assert index["index_type"] == get_simple_index["index_type"]
assert index["metric_type"] == get_simple_index["metric_type"]

11
tests/python_test/entity/test_insert.py
View File

@ -133,6 +133,7 @@ class TestInsertBase:
assert len(ids) == default_nb
connect.flush([collection])
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -147,6 +148,7 @@ class TestInsertBase:
connect.create_index(collection, field_name, get_simple_index)
ids = connect.insert(collection, default_entities)
assert len(ids) == default_nb
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -507,9 +509,10 @@ class TestInsertBase:
with pytest.raises(Exception):
connect.insert(collection, tmp_entity)
@pytest.mark.level(2)
@pytest.mark.timeout(30)
@pytest.mark.tags("0331")
# todo fix timeout
# @pytest.mark.level(2)
# @pytest.mark.timeout(30)
# @pytest.mark.tags("0331")
def test_collection_insert_rows_count_multi_threading(self, args, collection):
'''
target: test collection rows_count is correct or not with multi threading
@ -839,6 +842,7 @@ class TestInsertMultiCollections:
connect.create_index(collection, field_name, get_simple_index)
ids = connect.insert(collection_name, default_entity)
assert len(ids) == 1
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
connect.drop_collection(collection_name)
@ -856,6 +860,7 @@ class TestInsertMultiCollections:
ids = connect.insert(collection, default_entity)
connect.flush([collection])
connect.create_index(collection_name, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection_name, field_name)
assert index == get_simple_index
stats = connect.get_collection_stats(collection)

98
tests/python_test/entity/test_search.py
View File

@ -25,12 +25,12 @@ default_binary_query, default_binary_query_vecs = gen_query_vectors(binary_field
nq)
def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True):
def init_data(connect, collection, nb=3000, partition_tags=None, auto_id=True):
'''
Generate entities and add it in collection
'''
global entities
if nb == 1200:
if nb == 3000:
insert_entities = entities
else:
insert_entities = gen_entities(nb, is_normal=True)
@ -48,14 +48,14 @@ def init_data(connect, collection, nb=1200, partition_tags=None, auto_id=True):
return insert_entities, ids
def init_binary_data(connect, collection, nb=1200, insert=True, partition_tags=None):
def init_binary_data(connect, collection, nb=3000, insert=True, partition_tags=None):
'''
Generate entities and add it in collection
'''
ids = []
global binary_entities
global raw_vectors
if nb == 1200:
if nb == 3000:
insert_entities = binary_entities
insert_raw_vectors = raw_vectors
else:
@ -92,7 +92,7 @@ class TestSearchBase:
# if str(connect._cmd("mode")) == "CPU":
# if request.param["index_type"] in index_cpu_not_support():
# pytest.skip("sq8h not support in CPU mode")
return request.param
return copy.deepcopy(request.param)
@pytest.fixture(
scope="function",
@ -257,7 +257,6 @@ class TestSearchBase:
assert res2[0][0].id == res[0][1].id
assert res2[0][0].entity.get("int64") == res[0][1].entity.get("int64")
# TODO:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_after_index(self, connect, collection, get_simple_index, get_top_k, get_nq):
@ -287,7 +286,7 @@ class TestSearchBase:
assert res[0]._distances[0] < epsilon
assert check_id_result(res[0], ids[0])
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
def test_search_after_index_different_metric_type(self, connect, collection, get_simple_index):
'''
target: test search with different metric_type
@ -311,12 +310,7 @@ class TestSearchBase:
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
# TODO: need to enable
# description: create/load/search
# @pytest.mark.tags("0331")
def _test_search_after_index_different_metric_type_2(self, connect, collection, get_simple_index):
pass
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@ -350,7 +344,9 @@ class TestSearchBase:
res = connect.search(collection, query, partition_tags=[default_tag])
assert len(res[0]) == 0
@pytest.mark.tags("0331")
@pytest.mark.level(2)
@pytest.mark.timeout(600)
def test_search_index_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
target: test basic search function, all the search params is correct, test all index params, and build
@ -368,18 +364,19 @@ class TestSearchBase:
connect.create_index(collection, field_name, get_simple_index)
search_param = get_search_param(index_type)
query, vecs = gen_query_vectors(field_name, entities, top_k, nq, search_params=search_param)
for tags in [[default_tag], [default_tag, "new_tag"]]:
if top_k > max_top_k:
with pytest.raises(Exception) as e:
res = connect.search(collection, query, partition_tags=tags)
res = connect.search(collection, query, partition_tags=[default_tag])
else:
connect.load_partitions(collection, tags)
res = connect.search(collection, query, partition_tags=tags)
connect.load_partitions(collection, [default_tag])
res = connect.search(collection, query, partition_tags=[default_tag])
assert len(res) == nq
assert len(res[0]) >= top_k
assert len(res[0]) == top_k
assert res[0]._distances[0] < epsilon
assert check_id_result(res[0], ids[0])
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_index_partition_not_existed(self, connect, collection, get_top_k, get_nq, get_simple_index):
'''
@ -396,10 +393,9 @@ class TestSearchBase:
with pytest.raises(Exception) as e:
res = connect.search(collection, query, partition_tags=["new_tag"])
else:
connect.load_partitions(collection, ["new_tag"])
res = connect.search(collection, query, partition_tags=["new_tag"])
assert len(res) == nq
assert len(res[0]) == 0
connect.load_collection(collection)
with pytest.raises(Exception) as e:
connect.search(collection, query, partition_tags=["new_tag"])
@pytest.mark.tags("0331")
@pytest.mark.level(2)
@ -473,10 +469,7 @@ class TestSearchBase:
assert res[1]._distances[0] < epsilon
connect.release_collection(collection)
#
# test for ip metric
#
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_ip_flat(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@ -520,7 +513,7 @@ class TestSearchBase:
assert check_id_result(res[0], ids[0])
assert res[0]._distances[0] >= 1 - gen_inaccuracy(res[0]._distances[0])
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_ip_index_empty_partition(self, connect, collection, get_simple_index, get_top_k, get_nq):
'''
@ -630,8 +623,7 @@ class TestSearchBase:
res = connect.search(collection, query)
assert abs(np.sqrt(res[0]._distances[0]) - min(distance_0, distance_1)) <= gen_inaccuracy(res[0]._distances[0])
# TODO
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
def test_search_distance_l2_after_index(self, connect, id_collection, get_simple_index):
'''
target: search collection, and check the result: distance
@ -662,7 +654,7 @@ class TestSearchBase:
# TODO:
# assert abs(np.sqrt(res[0]._distances[0]) - min_distance) <= tmp_epsilon
# @pytest.mark.tags("0331", "l1")
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_distance_ip(self, connect, collection):
'''
@ -685,7 +677,7 @@ class TestSearchBase:
res = connect.search(collection, query)
assert abs(res[0]._distances[0] - max(distance_0, distance_1)) <= epsilon
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
def test_search_distance_ip_after_index(self, connect, id_collection, get_simple_index):
'''
target: search collection, and check the result: distance
@ -769,7 +761,6 @@ class TestSearchBase:
res = connect.search(binary_collection, query)
assert abs(res[0][0].distance - min(distance_0, distance_1).astype(float)) <= epsilon
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_distance_substructure_flat_index(self, connect, binary_collection):
@ -850,7 +841,7 @@ class TestSearchBase:
assert res[1][0].id in ids
assert res[1][0].distance <= epsilon
# @pytest.mark.tags("0331")
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_search_distance_tanimoto_flat_index(self, connect, binary_collection):
'''
@ -870,7 +861,7 @@ class TestSearchBase:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
@pytest.mark.timeout(30)
@pytest.mark.timeout(300)
def test_search_concurrent_multithreads(self, connect, args):
'''
target: test concurrent search with multiprocessess
@ -906,7 +897,7 @@ class TestSearchBase:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
@pytest.mark.timeout(30)
@pytest.mark.timeout(300)
def test_search_concurrent_multithreads_single_connection(self, connect, args):
'''
target: test concurrent search with multiprocessess
@ -1071,6 +1062,7 @@ class TestSearchDSL(object):
******************************************************************
"""
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_value_not_in(self, connect, collection):
'''
@ -1087,7 +1079,7 @@ class TestSearchDSL(object):
assert len(res[0]) == 0
# TODO:
# TODO:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_value_all_in(self, connect, collection):
'''
@ -1103,7 +1095,7 @@ class TestSearchDSL(object):
assert len(res[0]) == 1
# TODO:
# TODO:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_values_not_in(self, connect, collection):
'''
@ -1120,6 +1112,7 @@ class TestSearchDSL(object):
assert len(res[0]) == 0
# TODO:
@pytest.mark.tags("0331")
def test_query_term_values_all_in(self, connect, collection):
'''
method: build query with vector and term expr, with all term can be filtered
@ -1139,6 +1132,7 @@ class TestSearchDSL(object):
assert result.id in ids[:limit]
# TODO:
@pytest.mark.tags("0331")
def test_query_term_values_parts_in(self, connect, collection):
'''
method: build query with vector and term expr, with parts of term can be filtered
@ -1155,7 +1149,7 @@ class TestSearchDSL(object):
assert len(res[0]) == default_top_k
# TODO:
# TODO:
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_values_repeat(self, connect, collection):
'''
@ -1209,7 +1203,6 @@ class TestSearchDSL(object):
******************************************************************
"""
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_key_error(self, connect, collection):
@ -1244,7 +1237,7 @@ class TestSearchDSL(object):
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_term_field_named_term(self, connect, collection):
'''
@ -1266,7 +1259,7 @@ class TestSearchDSL(object):
expr = {"must": [gen_default_vector_expr(default_query),
term_param]}
query = update_query_expr(default_query, expr=expr)
connect.load_collection(collection)
connect.load_collection(collection_term)
res = connect.search(collection_term, query)
assert len(res) == nq
assert len(res[0]) == default_top_k
@ -1293,7 +1286,6 @@ class TestSearchDSL(object):
******************************************************************
"""
# TODO
@pytest.mark.tags("0331", "l1")
def test_query_range_key_error(self, connect, collection):
'''
@ -1313,7 +1305,6 @@ class TestSearchDSL(object):
def get_invalid_range(self, request):
return request.param
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_range_wrong_format(self, connect, collection, get_invalid_range):
@ -1343,6 +1334,7 @@ class TestSearchDSL(object):
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_range_invalid_ranges(self, connect, collection):
'''
@ -1365,6 +1357,7 @@ class TestSearchDSL(object):
def get_valid_ranges(self, request):
return request.param
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_range_valid_ranges(self, connect, collection, get_valid_ranges):
'''
@ -1401,7 +1394,7 @@ class TestSearchDSL(object):
************************************************************************
"""
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_multi_term_has_common(self, connect, collection):
'''
@ -1418,7 +1411,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == default_top_k
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_multi_term_no_common(self, connect, collection):
'''
@ -1435,7 +1428,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == 0
# TODO
@pytest.mark.tags("0331")
def test_query_multi_term_different_fields(self, connect, collection):
'''
method: build query with multi range with same field, and ranges no common
@ -1452,7 +1445,6 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == 0
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_single_term_multi_fields(self, connect, collection):
@ -1469,7 +1461,7 @@ class TestSearchDSL(object):
with pytest.raises(Exception) as e:
res = connect.search(collection, query)
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_multi_range_has_common(self, connect, collection):
'''
@ -1486,7 +1478,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == default_top_k
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_multi_range_no_common(self, connect, collection):
'''
@ -1503,7 +1495,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == 0
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_multi_range_different_fields(self, connect, collection):
'''
@ -1520,7 +1512,6 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == 0
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_single_range_multi_fields(self, connect, collection):
@ -1543,7 +1534,7 @@ class TestSearchDSL(object):
******************************************************************
"""
# TODO
@pytest.mark.tags("0331")
@pytest.mark.level(2)
def test_query_single_term_range_has_common(self, connect, collection):
'''
@ -1560,7 +1551,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == default_top_k
# TODO
@pytest.mark.tags("0331")
def test_query_single_term_range_no_common(self, connect, collection):
'''
method: build query with single term single range
@ -1582,7 +1573,6 @@ class TestSearchDSL(object):
******************************************************************
"""
# TODO
@pytest.mark.tags("0331")
def test_query_multi_vectors_same_field(self, connect, collection):
'''

5
tests/python_test/requirements.txt
View File

@ -10,6 +10,9 @@ allure-pytest==2.7.0
pytest-print==0.2.1
pytest-level==0.1.1
pytest-xdist==2.2.1
pymilvus-distributed==0.0.50
pymilvus-distributed==0.0.54
pytest-rerunfailures==9.1.1
git+https://github.com/Projectplace/pytest-tags
ndg-httpsclient
pyopenssl
pyasn1

20
tests/python_test/test_index.py
View File

@ -57,6 +57,7 @@ class TestIndexBase:
'''
ids = connect.insert(collection, default_entities)
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -94,6 +95,7 @@ class TestIndexBase:
expected: return search success
'''
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -108,6 +110,7 @@ class TestIndexBase:
connect.create_partition(collection, default_tag)
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -123,6 +126,7 @@ class TestIndexBase:
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
connect.flush([collection])
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -169,6 +173,7 @@ class TestIndexBase:
def build(connect):
connect.create_index(collection, field_name, default_index)
if default_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == default_index
@ -209,6 +214,7 @@ class TestIndexBase:
connect.flush([collection])
stats = connect.get_collection_stats(collection)
assert stats["row_count"] == default_nb
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -223,6 +229,7 @@ class TestIndexBase:
'''
connect.create_index(collection, field_name, get_simple_index)
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -243,7 +250,8 @@ class TestIndexBase:
connect.release_collection(collection)
connect.load_collection(collection)
index = connect.describe_index(collection, field_name)
assert index == indexs[-1]
# assert index == indexs[-1]
assert not index # FLAT is the last index_type, drop all indexes in server
@pytest.mark.tags("0331", "l1")
@pytest.mark.timeout(BUILD_TIMEOUT)
@ -256,6 +264,7 @@ class TestIndexBase:
ids = connect.insert(collection, default_entities)
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -269,6 +278,7 @@ class TestIndexBase:
'''
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -284,6 +294,7 @@ class TestIndexBase:
ids = connect.insert(collection, default_entities, partition_tag=default_tag)
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -300,6 +311,7 @@ class TestIndexBase:
connect.flush([collection])
get_simple_index["metric_type"] = "IP"
connect.create_index(collection, field_name, get_simple_index)
if get_simple_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == get_simple_index
@ -339,6 +351,7 @@ class TestIndexBase:
def build(connect):
default_index["metric_type"] = "IP"
connect.create_index(collection, field_name, default_index)
if default_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == default_index
@ -380,6 +393,7 @@ class TestIndexBase:
connect.flush([collection])
stats = connect.get_collection_stats(collection)
assert stats["row_count"] == default_nb
if default_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == default_index
@ -395,6 +409,7 @@ class TestIndexBase:
default_index["metric_type"] = "IP"
connect.create_index(collection, field_name, default_index)
connect.create_index(collection, field_name, default_index)
if default_index["index_type"] != "FLAT":
index = connect.describe_index(collection, field_name)
assert index == default_index
@ -419,7 +434,8 @@ class TestIndexBase:
connect.release_collection(collection)
connect.load_collection(collection)
index = connect.describe_index(collection, field_name)
assert index == indexs[-1]
# assert index == indexs[-1]
assert not index
"""
******************************************************************