mirror of https://github.com/milvus-io/milvus.git
Fix the error that write node cannot receive the message
Signed-off-by: xige-16 <xi.ge@zilliz.com>pull/4973/head^2
xige-16
yefu.chen
parent
bfd4fe5a82
commit
c076bd4a73
|
|
@ -27,6 +27,10 @@ proxy/cmake_build
|
|||
proxy/cmake-build-debug
|
||||
proxy/thirdparty/grpc-src
|
||||
proxy/thirdparty/grpc-build
|
||||
proxy/milvus/
|
||||
proxy/milvus/*
|
||||
proxy/suvlim/
|
||||
proxy/suvlim/*
|
||||
|
||||
# Compiled source
|
||||
*.a
|
||||
|
|
|
|||
31
go.sum
31
go.sum
|
|
@ -44,6 +44,7 @@ github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af h1:
|
|||
github.com/apache/pulsar/pulsar-client-go v0.0.0-20200901051823-800681aaa9af/go.mod h1:QdYxU2iG99VVU6cvoBRkCgkazfJSL9WwPZ20PZR6aUk=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
|
|
@ -118,8 +119,10 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
|
|||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
|
||||
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
|
|
@ -140,6 +143,7 @@ github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hf
|
|||
github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
|
||||
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
||||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
|
|
@ -148,7 +152,9 @@ github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z
|
|||
github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 h1:z53tR0945TRRQO/fLEVPI6SMv7ZflF0TEaTAoU7tOzg=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.8.1/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
|
|
@ -160,6 +166,7 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt
|
|||
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
|
||||
|
|
@ -169,6 +176,7 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW
|
|||
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
|
||||
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
|
|
@ -176,18 +184,26 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
|
|||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/linkedin/goavro v2.1.0+incompatible h1:DV2aUlj2xZiuxQyvag8Dy7zjY69ENjS66bWkSfdpddY=
|
||||
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
|
||||
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
|
||||
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
|
||||
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
|
||||
github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
|
||||
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
|
||||
|
|
@ -196,6 +212,7 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
|
|||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
|
||||
github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 h1:R8gStypOBmpnHEx1qi//SaqxJVI4inOqljg/Aj5/390=
|
||||
|
|
@ -204,29 +221,37 @@ github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw
|
|||
github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
|
||||
github.com/pingcap/gofail v0.0.0-20181217135706-6a951c1e42c3/go.mod h1:DazNTg0PTldtpsQiT9I5tVJwV1onHMKBBgXzmJUlMns=
|
||||
github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw=
|
||||
github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9 h1:EsTt42btov+tFchxOFKnxBNmXOWyPKiddOwvr/WO90g=
|
||||
github.com/pingcap/kvproto v0.0.0-20190305055742-ab7debc182d9/go.mod h1:QMdbTAXCHzzygQzqcG9uVUgU2fKeSN1GmfMiykdSzzY=
|
||||
github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
|
||||
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 h1:Jboj+s4jSCp5E1WDgmRUv5rIFKFHaaSWuSZ4wMwXIcc=
|
||||
github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
|
||||
github.com/pingcap/pd v2.1.5+incompatible h1:vOLV2tSQdRjjmxaTXtJULoC94dYQOd+6fzn2yChODHc=
|
||||
github.com/pingcap/pd v2.1.5+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E=
|
||||
github.com/pivotal-golang/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:43j3yLP9UiXa0z95/W3hN7yTjoxsQoOll5rrGBgBcnE=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM=
|
||||
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw=
|
||||
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
|
||||
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs=
|
||||
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
|
|
@ -244,6 +269,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b h1:VOG2GkM7RpRrT0St7HIIwCWrc3mVdf+DjcT8r2ucusI=
|
||||
github.com/tikv/client-go v0.0.0-20200824032810-95774393107b/go.mod h1:K0NcdVNrXDq92YPLytsrAwRMyuXi7GZCO6dXNH7OzQc=
|
||||
github.com/tikv/pd v2.1.19+incompatible/go.mod h1:v6C/D7ONC49SgjI4jbGnooSizvijaO/bdIm62DVR4tI=
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
|
||||
|
|
@ -285,6 +311,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
|
|||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
|
|
@ -347,6 +374,7 @@ golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/
|
|||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA=
|
||||
golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
|
|
@ -398,6 +426,7 @@ golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fq
|
|||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
|
@ -504,6 +533,7 @@ google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEY
|
|||
google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA=
|
||||
google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98 h1:LCO0fg4kb6WwkXQXRQQgUYsFeFb5taTX5WAx5O/Vt28=
|
||||
google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
|
||||
google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
|
|
@ -539,6 +569,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
|
|||
gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw=
|
||||
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/linkedin/goavro.v1 v1.0.5/go.mod h1:Aw5GdAbizjOEl0kAMHV9iHmA8reZzW/OKuJAl4Hb9F0=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
|
||||
|
|
|
|||
Binary file not shown.
|
|
@ -25,7 +25,8 @@ add_subdirectory( query )
|
|||
add_subdirectory( db ) # target milvus_engine
|
||||
add_subdirectory( log )
|
||||
add_subdirectory( server )
|
||||
add_subdirectory( pulsar )
|
||||
add_subdirectory( message_client )
|
||||
add_subdirectory( test )
|
||||
|
||||
set(link_lib
|
||||
milvus_engine
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@
|
|||
#include "src/version.h"
|
||||
#include "utils/SignalHandler.h"
|
||||
#include "utils/Status.h"
|
||||
#include "pulsar/message_client/Client.h"
|
||||
#include "message_client/Client.h"
|
||||
|
||||
INITIALIZE_EASYLOGGINGPP
|
||||
|
||||
|
|
|
|||
|
|
@ -3,11 +3,14 @@ set(src-cpp
|
|||
ClientV2.cpp
|
||||
Consumer.cpp
|
||||
Producer.cpp
|
||||
../grpc/gen-milvus/suvlim.pb.cc
|
||||
../grpc/gen-status/status.pb.cc
|
||||
)
|
||||
|
||||
add_library(message_client_cpp SHARED
|
||||
add_library(message_client_cpp
|
||||
${src-cpp}
|
||||
)
|
||||
|
||||
target_include_directories(message_client_cpp PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar/pulsar-src/pulsar-client-cpp/include)
|
||||
|
||||
target_link_libraries(message_client_cpp pulsarStatic libprotobuf)
|
||||
|
|
@ -21,14 +21,19 @@ MsgClientV2 &MsgClientV2::GetInstance() {
|
|||
MsgClientV2::MsgClientV2(int64_t client_id, std::string &service_url, const pulsar::ClientConfiguration &config)
|
||||
: client_id_(client_id), service_url_(service_url) {}
|
||||
|
||||
Status MsgClientV2::Init(const std::string &mut_topic, const std::string &query_topic,
|
||||
const std::string &result_topic) {
|
||||
auto pulsar_client = std::make_shared<pulsar::Client>(service_url_);
|
||||
mut_producer_ = std::make_shared<MsgProducer>(pulsar_client, mut_topic);
|
||||
query_producer_ = std::make_shared<MsgProducer>(pulsar_client, mut_topic);
|
||||
consumer_ = std::make_shared<MsgConsumer>(pulsar_client, result_topic);
|
||||
Status MsgClientV2::Init(const std::string &insert_delete,
|
||||
const std::string &search,
|
||||
const std::string &time_sync,
|
||||
const std::string &search_by_id,
|
||||
const std::string &search_result) {
|
||||
auto pulsar_client = std::make_shared<MsgClient>(service_url_);
|
||||
insert_delete_producer_ = std::make_shared<MsgProducer>(pulsar_client, insert_delete);
|
||||
search_producer_ = std::make_shared<MsgProducer>(pulsar_client, search);
|
||||
search_by_id_producer_ = std::make_shared<MsgProducer>(pulsar_client, search_by_id);
|
||||
time_sync_producer_ = std::make_shared<MsgProducer>(pulsar_client, time_sync);
|
||||
consumer_ = std::make_shared<MsgConsumer>(pulsar_client, search_result);
|
||||
|
||||
auto result = consumer_->subscribe(result_topic);
|
||||
auto result = consumer_->subscribe(search_result);
|
||||
if (result != pulsar::Result::ResultOk) {
|
||||
return Status(SERVER_UNEXPECTED_ERROR, "Pulsar message client init occur error, " + std::string(pulsar::strResult(result)));
|
||||
}
|
||||
|
|
@ -57,7 +62,7 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::InsertParam &request) {
|
|||
mut_msg.mutable_rows_data()->CopyFrom(request.rows_data(i));
|
||||
mut_msg.mutable_extra_params()->CopyFrom(request.extra_params());
|
||||
|
||||
auto result = mut_producer_->send(mut_msg);
|
||||
auto result = insert_delete_producer_->send(mut_msg);
|
||||
if (result != pulsar::ResultOk) {
|
||||
// TODO: error code
|
||||
return Status(DB_ERROR, pulsar::strResult(result));
|
||||
|
|
@ -75,7 +80,7 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request)
|
|||
mut_msg.set_uid(id);
|
||||
mut_msg.set_collection_name(request.collection_name());
|
||||
|
||||
auto result = mut_producer_->send(mut_msg);
|
||||
auto result = insert_delete_producer_->send(mut_msg);
|
||||
if (result != pulsar::ResultOk) {
|
||||
// TODO: error code
|
||||
return Status(DB_ERROR, pulsar::strResult(result));
|
||||
|
|
@ -85,8 +90,10 @@ Status MsgClientV2::SendMutMessage(const milvus::grpc::DeleteByIDParam &request)
|
|||
}
|
||||
|
||||
MsgClientV2::~MsgClientV2() {
|
||||
mut_producer_->close();
|
||||
query_producer_->close();
|
||||
insert_delete_producer_->close();
|
||||
search_producer_->close();
|
||||
search_by_id_producer_->close();
|
||||
time_sync_producer_->close();
|
||||
consumer_->close();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#pragma once
|
||||
|
||||
#include "src/utils/Status.h"
|
||||
#include "utils/Status.h"
|
||||
#include "Producer.h"
|
||||
#include "Consumer.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
|
|
@ -12,8 +12,11 @@ class MsgClientV2 {
|
|||
~MsgClientV2();
|
||||
|
||||
// When using MsgClient, make sure it init successfully
|
||||
Status Init(const std::string &mut_topic,
|
||||
const std::string &query_topic, const std::string &result_topic);
|
||||
Status Init(const std::string &insert_delete,
|
||||
const std::string &search,
|
||||
const std::string &time_sync,
|
||||
const std::string &search_by_id,
|
||||
const std::string &search_result);
|
||||
|
||||
// unpackage batch insert or delete request, and delivery message to pulsar per row
|
||||
Status SendMutMessage(const milvus::grpc::InsertParam &request);
|
||||
|
|
@ -40,7 +43,9 @@ class MsgClientV2 {
|
|||
int64_t client_id_;
|
||||
std::string service_url_;
|
||||
std::shared_ptr<MsgConsumer> consumer_;
|
||||
std::shared_ptr<MsgProducer> mut_producer_;
|
||||
std::shared_ptr<MsgProducer> query_producer_;
|
||||
std::shared_ptr<MsgProducer> insert_delete_producer_;
|
||||
std::shared_ptr<MsgProducer> search_producer_;
|
||||
std::shared_ptr<MsgProducer> time_sync_producer_;
|
||||
std::shared_ptr<MsgProducer> search_by_id_producer_;
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
|
||||
#include "Consumer.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
MsgConsumer::MsgConsumer(std::shared_ptr<MsgClient> &client, std::string subscription_name, const ConsumerConfiguration conf)
|
||||
:client_(client), config_(conf), subscription_name_(subscription_name){}
|
||||
|
||||
Result MsgConsumer::subscribe(const std::string &topic) {
|
||||
return client_->subscribe(topic, subscription_name_, config_, consumer_);
|
||||
}
|
||||
|
||||
Result MsgConsumer::subscribe(const std::vector<std::string> &topics) {
|
||||
return client_->subscribe(topics, subscription_name_, config_, consumer_);
|
||||
}
|
||||
|
||||
Result MsgConsumer::unsubscribe() {
|
||||
return consumer_.unsubscribe();
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(Message &msg) {
|
||||
return consumer_.receive(msg);
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(milvus::grpc::QueryResult &res) {
|
||||
Message msg;
|
||||
auto result = consumer_.receive(msg);
|
||||
if (result == pulsar::ResultOk) {
|
||||
res.ParseFromString(msg.getDataAsString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(milvus::grpc::Entities &res) {
|
||||
Message msg;
|
||||
auto result = consumer_.receive(msg);
|
||||
if (result == pulsar::ResultOk) {
|
||||
res.ParseFromString(msg.getDataAsString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(milvus::grpc::EntityIds &res) {
|
||||
Message msg;
|
||||
auto result = consumer_.receive(msg);
|
||||
if (result == pulsar::ResultOk) {
|
||||
res.ParseFromString(msg.getDataAsString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(milvus::grpc::Status &res) {
|
||||
Message msg;
|
||||
auto result = consumer_.receive(msg);
|
||||
if (result == pulsar::ResultOk) {
|
||||
res.ParseFromString(msg.getDataAsString());
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
Result MsgConsumer::close() {
|
||||
return consumer_.close();
|
||||
}
|
||||
|
||||
Result MsgConsumer::acknowledge(const Message &message) {
|
||||
return consumer_.acknowledge(message);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,15 +1,17 @@
|
|||
#pragma once
|
||||
|
||||
#include "pulsar/Consumer.h"
|
||||
#include "client.h"
|
||||
#include "Client.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
#include "grpc/gen-status/status.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
enum ConsumerType {
|
||||
INSERT = 0,
|
||||
DELETE = 1,
|
||||
SEARCH_RESULT = 2,
|
||||
TEST = 3,
|
||||
};
|
||||
|
||||
using Consumer = pulsar::Consumer;
|
||||
|
|
@ -17,14 +19,17 @@ using ConsumerConfiguration = pulsar::ConsumerConfiguration;
|
|||
|
||||
class MsgConsumer{
|
||||
public:
|
||||
MsgConsumer(std::shared_ptr<message_client::MsgClient> &client, std::string consumer_name,
|
||||
MsgConsumer(std::shared_ptr<MsgClient> &client, std::string consumer_name,
|
||||
const pulsar::ConsumerConfiguration conf = ConsumerConfiguration());
|
||||
|
||||
Result subscribe(const std::string& topic);
|
||||
Result subscribe(const std::vector<std::string>& topics);
|
||||
Result unsubscribe();
|
||||
Result receive(Message& msg);
|
||||
std::shared_ptr<void> receive_proto(ConsumerType consumer_type);
|
||||
Result receive(milvus::grpc::QueryResult &res);
|
||||
Result receive(milvus::grpc::EntityIds &res);
|
||||
Result receive(milvus::grpc::Entities &res);
|
||||
Result receive(milvus::grpc::Status &res);
|
||||
Result acknowledge(const Message& message);
|
||||
Result close();
|
||||
|
||||
|
|
@ -33,9 +38,10 @@ public:
|
|||
|
||||
private:
|
||||
Consumer consumer_;
|
||||
std::shared_ptr<MsgClient> client_;
|
||||
std::shared_ptr<pulsar::Client> client_;
|
||||
ConsumerConfiguration config_;
|
||||
std::string subscription_name_;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
|
||||
#include "Producer.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
MsgProducer::MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic,
|
||||
const ProducerConfiguration& conf) : client_(client), config_(conf) {
|
||||
createProducer(topic);
|
||||
}
|
||||
|
||||
Result MsgProducer::createProducer(const std::string &topic) {
|
||||
return client_->createProducer(topic, producer_);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const Message &msg) {
|
||||
return producer_.send(msg);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const std::string &msg) {
|
||||
auto pulsar_msg = pulsar::MessageBuilder().setContent(msg).build();
|
||||
return send(pulsar_msg);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const milvus::grpc::InsertOrDeleteMsg &msg) {
|
||||
auto msg_str = msg.SerializeAsString();
|
||||
return send(msg_str);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const milvus::grpc::SearchMsg &msg) {
|
||||
auto msg_str = msg.SerializeAsString();
|
||||
return send(msg_str);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const milvus::grpc::GetEntityIDsParam &msg) {
|
||||
auto msg_str = msg.SerializeAsString();
|
||||
return send(msg_str);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const milvus::grpc::TimeSyncMsg &msg) {
|
||||
auto msg_str = msg.SerializeAsString();
|
||||
return send(msg_str);
|
||||
}
|
||||
|
||||
Result MsgProducer::close() {
|
||||
return producer_.close();
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#pragma once
|
||||
|
||||
#include "pulsar/Producer.h"
|
||||
#include "Client.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
using Producer = pulsar::Producer;
|
||||
using ProducerConfiguration = pulsar::ProducerConfiguration;
|
||||
|
||||
class MsgProducer {
|
||||
public:
|
||||
MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic,
|
||||
const ProducerConfiguration &conf = ProducerConfiguration());
|
||||
|
||||
Result createProducer(const std::string &topic);
|
||||
Result send(const Message &msg);
|
||||
Result send(const std::string &msg);
|
||||
Result send(const milvus::grpc::InsertOrDeleteMsg &msg);
|
||||
Result send(const milvus::grpc::SearchMsg &msg);
|
||||
Result send(const milvus::grpc::GetEntityIDsParam &msg);
|
||||
Result send(const milvus::grpc::TimeSyncMsg & msg);
|
||||
Result close();
|
||||
|
||||
const Producer &
|
||||
producer() const { return producer_; }
|
||||
|
||||
private:
|
||||
Producer producer_;
|
||||
std::shared_ptr<pulsar::Client> client_;
|
||||
ProducerConfiguration config_;
|
||||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
add_subdirectory(message_client)
|
||||
#add_subdirectory(unittest)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
enable_testing()
|
||||
set(unittest_srcs
|
||||
unittest_entry.cpp
|
||||
consumer_test.cpp producer_test.cpp)
|
||||
|
||||
add_executable(test ${unittest_srcs})
|
||||
|
||||
target_link_libraries(test
|
||||
message_client_cpp
|
||||
pulsar
|
||||
gtest
|
||||
gtest_main)
|
||||
|
||||
install(TARGETS test DESTINATION unittest)
|
||||
|
|
@ -1,14 +0,0 @@
|
|||
#include <gtest/gtest.h>
|
||||
#include "pulsar/message_client/Consumer.h"
|
||||
#include "pulsar/message_client/pb/pulsar.pb.h"
|
||||
|
||||
TEST(CLIENT_CPP, CONSUMER) {
|
||||
auto client= std::make_shared<message_client::MsgClient>("pulsar://localhost:6650");
|
||||
message_client::MsgConsumer consumer(client, "my_consumer");
|
||||
consumer.subscribe("test");
|
||||
auto msg = consumer.receive_proto(message_client::TEST);
|
||||
pb::TestData* data = (pb::TestData*)(msg.get());
|
||||
std::cout << "Received: " << msg << " with payload '" << data->name()<< ";" << data->id();
|
||||
consumer.close();
|
||||
client->close();
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
#include <gtest/gtest.h>
|
||||
#include "pulsar/message_client/Producer.h"
|
||||
#include "pulsar/message_client/pb/pulsar.pb.h"
|
||||
|
||||
TEST(CLIENT_CPP, Producer) {
|
||||
auto client= std::make_shared<message_client::MsgClient>("pulsar://localhost:6650");
|
||||
message_client::MsgProducer producer(client,"test");
|
||||
pb::TestData data;
|
||||
data.set_id("test");
|
||||
data.set_name("hahah");
|
||||
std::string to_string = data.SerializeAsString();
|
||||
producer.send(to_string);
|
||||
producer.close();
|
||||
client->close();
|
||||
}
|
||||
|
|
@ -26,6 +26,7 @@ aux_source_directory( ${MILVUS_ENGINE_SRC}/server/init SERVER_INIT_
|
|||
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/delivery/request DELIVERY_REQUEST_FILES )
|
||||
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/delivery/strategy DELIVERY_STRATEGY_FILES )
|
||||
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/delivery DELIVERY_FILES )
|
||||
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/timesync TIME_SYNC_FILES )
|
||||
aux_source_directory( ${MILVUS_ENGINE_SRC}/server/tso TSO_FILES)
|
||||
|
||||
set( SERVER_FILES ${SERVER_INIT_FILES}
|
||||
|
|
@ -34,6 +35,7 @@ set( SERVER_FILES ${SERVER_INIT_FILES}
|
|||
${DELIVERY_REQUEST_FILES}
|
||||
${DELIVERY_STRATEGY_FILES}
|
||||
${DELIVERY_FILES}
|
||||
${TIME_SYNC_FILES}
|
||||
${TSO_FILES}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@
|
|||
#include "server/init/StorageChecker.h"
|
||||
#include "src/version.h"
|
||||
#include <yaml-cpp/yaml.h>
|
||||
#include "src/pulsar/message_client/ClientV2.h"
|
||||
#include "message_client/ClientV2.h"
|
||||
#include "utils/Log.h"
|
||||
#include "utils/SignalHandler.h"
|
||||
#include "utils/TimeRecorder.h"
|
||||
|
|
@ -291,7 +291,7 @@ Server::StartService() {
|
|||
|
||||
grpc::GrpcServer::GetInstance().Start();
|
||||
|
||||
stat = message_client::MsgClientV2::GetInstance().Init("topic-insert","topic-query","topic-result");
|
||||
stat = message_client::MsgClientV2::GetInstance().Init("InsertOrDelete", "Search", "TimeSync", "SearchById", "SearchResult");
|
||||
if (!stat.ok()) {
|
||||
LOG_SERVER_ERROR_ << "Pulsar message client start service fail: " << stat.message();
|
||||
goto FAIL;
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
#include "server/context/Context.h"
|
||||
#include "server/delivery/request/Types.h"
|
||||
#include "utils/Status.h"
|
||||
#include "pulsar/message_client/ClientV2.h"
|
||||
#include "message_client/ClientV2.h"
|
||||
|
||||
#include <condition_variable>
|
||||
#include <memory>
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@
|
|||
#include "server/delivery/ReqHandler.h"
|
||||
#include "server/grpc_impl/interceptor/GrpcInterceptorHookHandler.h"
|
||||
#include "src/utils/Status.h"
|
||||
#include "pulsar/message_client/Consumer.h"
|
||||
#include "pulsar/message_client/Producer.h"
|
||||
#include "pulsar/message_client/ClientV2.h"
|
||||
#include "message_client/Consumer.h"
|
||||
#include "message_client/Producer.h"
|
||||
#include "message_client/ClientV2.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@
|
|||
// #include "server/DBWrapper.h"
|
||||
#include "server/grpc_impl/interceptor/SpanInterceptor.h"
|
||||
#include "utils/Log.h"
|
||||
#include "pulsar/message_client/ClientV2.h"
|
||||
#include "message_client/ClientV2.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace server {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include <chrono>
|
||||
#include "TimeSync.h"
|
||||
#include "message_client/Producer.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace timesync {
|
||||
|
||||
TimeSync::TimeSync(int64_t id,
|
||||
std::function<uint64_t()> timestamp,
|
||||
const int interval,
|
||||
const std::string &pulsar_addr,
|
||||
const std::string &time_sync_topic) :
|
||||
timestamp_(timestamp), interval_(interval), pulsar_addr_(pulsar_addr), time_sync_topic_(time_sync_topic) {
|
||||
sync_msg_.set_peer_id(id);
|
||||
auto timer = [&]() {
|
||||
std::shared_ptr<milvus::message_client::MsgClient>
|
||||
client = std::make_shared<milvus::message_client::MsgClient>(this->pulsar_addr_);
|
||||
milvus::message_client::MsgProducer producer(client, this->time_sync_topic_);
|
||||
|
||||
for (;;) {
|
||||
if (this->stop_) break;
|
||||
this->sync_msg_.set_timestamp(this->timestamp_());
|
||||
this->sync_msg_.set_sync_type(milvus::grpc::READ);
|
||||
auto rst = producer.send(sync_msg_.SerializeAsString());
|
||||
if (rst != pulsar::ResultOk) {
|
||||
//TODO, add log
|
||||
}
|
||||
std::this_thread::sleep_for(std::chrono::milliseconds(this->interval_));
|
||||
}
|
||||
auto rst = producer.close();
|
||||
if (rst != pulsar::ResultOk) {
|
||||
//TODO, add log or throw exception
|
||||
}
|
||||
rst = client->close();
|
||||
if (rst != pulsar::ResultOk) {
|
||||
//TODO, add log or throw exception
|
||||
}
|
||||
};
|
||||
timer_ = std::thread(timer);
|
||||
}
|
||||
|
||||
TimeSync::~TimeSync() {
|
||||
stop_ = true;
|
||||
timer_.join();
|
||||
}
|
||||
|
||||
void TimeSync::Stop() {
|
||||
stop_ = true;
|
||||
}
|
||||
|
||||
bool TimeSync::IsStop() const {
|
||||
return stop_;
|
||||
}
|
||||
|
||||
} // namespace timesync
|
||||
} // namespace milvus
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <cstdint>
|
||||
#include <thread>
|
||||
#include <string>
|
||||
#include "suvlim.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace timesync {
|
||||
|
||||
class TimeSync {
|
||||
public:
|
||||
TimeSync(int64_t id,
|
||||
std::function<uint64_t()> timestamp,
|
||||
const int interval,
|
||||
const std::string &pulsar_addr,
|
||||
const std::string &time_sync_topic);
|
||||
virtual ~TimeSync();
|
||||
|
||||
void Stop();
|
||||
bool IsStop() const;
|
||||
private:
|
||||
std::function<int64_t()> timestamp_;
|
||||
const int interval_;
|
||||
const std::string pulsar_addr_;
|
||||
const std::string time_sync_topic_;
|
||||
bool stop_ = false;
|
||||
std::thread timer_;
|
||||
milvus::grpc::TimeSyncMsg sync_msg_;
|
||||
};
|
||||
|
||||
} // namespace timesync
|
||||
} // namespace milvus
|
||||
|
|
@ -1,26 +1,26 @@
|
|||
AUX_SOURCE_DIRECTORY(. TEST)
|
||||
#AUX_SOURCE_DIRECTORY(. TEST)
|
||||
|
||||
set( GRPC_SERVICE_FILES ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/hello.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/hello.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/master.grpc.pb.cc
|
||||
${MILVUS_ENGINE_SRC}/grpc/gen-milvus/master.pb.cc
|
||||
)
|
||||
add_executable(test_pulsar ${TEST} ${GRPC_SERVICE_FILES})
|
||||
#set( GRPC_SERVICE_FILES ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.grpc.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/milvus.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.grpc.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-status/status.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/hello.grpc.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/hello.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/master.grpc.pb.cc
|
||||
# ${MILVUS_ENGINE_SRC}/grpc/gen-milvus/master.pb.cc
|
||||
# )
|
||||
#add_executable(test_pulsar ${TEST} ${GRPC_SERVICE_FILES})
|
||||
#
|
||||
#target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar/pulsar-src/pulsar-client-cpp/include)
|
||||
#target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/avro/avro-build/include)
|
||||
|
||||
target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar/pulsar-src/pulsar-client-cpp/include)
|
||||
target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/avro/avro-build/include)
|
||||
|
||||
|
||||
target_link_libraries(test_pulsar
|
||||
pulsarStatic
|
||||
libprotobuf
|
||||
grpc++_reflection
|
||||
grpc++
|
||||
libboost_system.a
|
||||
libboost_filesystem.a
|
||||
libboost_serialization.a
|
||||
)
|
||||
add_subdirectory(message_client_test)
|
||||
#target_link_libraries(test_pulsar
|
||||
# pulsarStatic
|
||||
# libprotobuf
|
||||
# grpc++_reflection
|
||||
# grpc++
|
||||
# libboost_system.a
|
||||
# libboost_filesystem.a
|
||||
# libboost_serialization.a
|
||||
# )
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
enable_testing()
|
||||
set(unittest_srcs
|
||||
unittest_entry.cpp
|
||||
consumer_test.cpp producer_test.cpp)
|
||||
|
||||
add_executable(test_pulsar ${unittest_srcs})
|
||||
|
||||
target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar/pulsar-src/pulsar-client-cpp/include)
|
||||
target_include_directories(test_pulsar PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/avro/avro-build/include)
|
||||
|
||||
#add_executable(test_pulsar ${unittest_srcs})
|
||||
|
||||
target_link_libraries(test_pulsar
|
||||
message_client_cpp
|
||||
pulsarStatic
|
||||
gtest
|
||||
gtest_main
|
||||
libprotobuf
|
||||
grpc++_reflection
|
||||
grpc++
|
||||
libboost_system.a
|
||||
libboost_filesystem.a
|
||||
libboost_serialization.a
|
||||
)
|
||||
|
||||
install(TARGETS test_pulsar DESTINATION unittest)
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
#include <gtest/gtest.h>
|
||||
#include "message_client/Consumer.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
#include "grpc/gen-status/status.pb.h"
|
||||
|
||||
TEST(CLIENT_CPP, CONSUMER) {
|
||||
auto client= std::make_shared<milvus::message_client::MsgClient>("pulsar://localhost:6650");
|
||||
milvus::message_client::MsgConsumer consumer(client, "my_consumer");
|
||||
consumer.subscribe("test");
|
||||
milvus::grpc::Status msg;
|
||||
auto res = consumer.receive(msg);
|
||||
// pb::TestData* data = (pb::TestData*)(msg.get());
|
||||
std::cout << "Received: with payload reason" << msg.reason();
|
||||
consumer.close();
|
||||
client->close();
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
#include <gtest/gtest.h>
|
||||
#include "message_client/Producer.h"
|
||||
#include "grpc/gen-milvus/suvlim.pb.h"
|
||||
#include "grpc/gen-status/status.pb.h"
|
||||
|
||||
TEST(CLIENT_CPP, Producer) {
|
||||
auto client= std::make_shared<milvus::message_client::MsgClient>("pulsar://localhost:6650");
|
||||
milvus::message_client::MsgProducer producer(client,"test");
|
||||
milvus::grpc::Status msg;
|
||||
msg.set_error_code(::milvus::grpc::SUCCESS);
|
||||
msg.set_reason("no reason");
|
||||
std::string to_string = msg.SerializeAsString();
|
||||
producer.send(to_string);
|
||||
producer.close();
|
||||
client->close();
|
||||
}
|
||||
|
||||
TEST(CLIENT_CPP, PRODUCE_INSERT) {
|
||||
auto client= std::make_shared<milvus::message_client::MsgClient>("pulsar://localhost:6650");
|
||||
milvus::message_client::MsgProducer producer(client,"InsertOrDelete");
|
||||
int64_t offset = 1;
|
||||
milvus::grpc::RowData data;
|
||||
milvus::grpc::InsertOrDeleteMsg msg;
|
||||
while (offset <= 100000) {
|
||||
data.set_blob("a blob");
|
||||
msg.set_collection_name("zilliz");
|
||||
msg.set_partition_tag("milvus");
|
||||
msg.set_segment_id(0);
|
||||
msg.set_channel_id(0);
|
||||
msg.set_client_id(0);
|
||||
msg.set_uid(offset);
|
||||
msg.set_timestamp(offset);
|
||||
msg.set_op(milvus::grpc::INSERT);
|
||||
|
||||
std::string to_string = msg.SerializeAsString();
|
||||
producer.send(to_string);
|
||||
if (offset % 20 == 0)
|
||||
usleep(200000);
|
||||
offset++;
|
||||
}
|
||||
// producer.close();
|
||||
client->close();
|
||||
}
|
||||
|
|
@ -11,6 +11,5 @@
|
|||
|
||||
#define MILVUS_VERSION "0.10.0"
|
||||
#define BUILD_TYPE "Debug"
|
||||
|
||||
#define BUILD_TIME "2020-09-03 16:41.13"
|
||||
#define LAST_COMMIT_ID "a97d3e925e993f4a2a7409a9734dcc788f109767"
|
||||
#define BUILD_TIME "2020-09-06 15:30.38"
|
||||
#define LAST_COMMIT_ID "f569a5f01205b6b2eb9d8fb7442c5c12c57309fe"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
set(src-cpp
|
||||
client.cpp
|
||||
consumer.cpp
|
||||
producer.cpp
|
||||
pb/pulsar.pb.cc)
|
||||
Consumer.cpp
|
||||
Producer.cpp
|
||||
Client.cpp
|
||||
pb/suvlim.pb.cc
|
||||
pb/status.pb.cc)
|
||||
|
||||
add_library(client_cpp SHARED
|
||||
${src-cpp}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
|
||||
#include "client.h"
|
||||
#include "Client.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
MsgClient::MsgClient(const std::string &serviceUrl) : pulsar::Client(serviceUrl) {}
|
||||
|
||||
MsgClient::MsgClient(const std::string &serviceUrl, const pulsar::ClientConfiguration& clientConfiguration)
|
||||
: pulsar::Client(serviceUrl, clientConfiguration) {}
|
||||
MsgClient::MsgClient(const std::string &serviceUrl, const pulsar::ClientConfiguration &clientConfiguration)
|
||||
: pulsar::Client(serviceUrl, clientConfiguration) {}
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
#pragma once
|
||||
|
||||
#include "pulsar/Client.h"
|
||||
#include <pulsar/ClientConfiguration.h>
|
||||
#include "pulsar/ClientConfiguration.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
||||
using Result = pulsar::Result;
|
||||
|
|
@ -22,3 +23,4 @@ private:
|
|||
};
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
|
||||
#include "Consumer.h"
|
||||
#include "src/grpc/gen-milvus/suvlim.pb.h"
|
||||
#include "pb/suvlim.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
#include "pulsar/Producer.h"
|
||||
#include "Client.h"
|
||||
#include "src/grpc/gen-milvus/suvlim.pb.h"
|
||||
#include "pb/suvlim.pb.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace message_client {
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
|
||||
#include "consumer.h"
|
||||
#include "pb/pulsar.pb.h"
|
||||
|
||||
namespace message_client {
|
||||
|
||||
MsgConsumer::MsgConsumer(std::shared_ptr<MsgClient> &client, std::string subscription_name, const ConsumerConfiguration conf)
|
||||
:client_(client), config_(conf), subscription_name_(subscription_name){}
|
||||
|
||||
Result MsgConsumer::subscribe(const std::string &topic) {
|
||||
return client_->subscribe(topic, subscription_name_, config_, consumer_);
|
||||
}
|
||||
|
||||
Result MsgConsumer::subscribe(const std::vector<std::string> &topics) {
|
||||
return client_->subscribe(topics, subscription_name_, config_, consumer_);
|
||||
}
|
||||
|
||||
Result MsgConsumer::unsubscribe() {
|
||||
return consumer_.unsubscribe();
|
||||
}
|
||||
|
||||
Result MsgConsumer::receive(Message &msg) {
|
||||
return consumer_.receive(msg);
|
||||
}
|
||||
|
||||
std::shared_ptr<void> MsgConsumer::receive_proto(ConsumerType consumer_type) {
|
||||
Message msg;
|
||||
receive(msg);
|
||||
acknowledge(msg);
|
||||
switch (consumer_type) {
|
||||
case INSERT: {
|
||||
pb::InsertMsg insert_msg;
|
||||
insert_msg.ParseFromString(msg.getDataAsString());
|
||||
auto message = std::make_shared<pb::InsertMsg>(insert_msg);
|
||||
return std::shared_ptr<void>(message);
|
||||
}
|
||||
case DELETE: {
|
||||
pb::DeleteMsg delete_msg;
|
||||
delete_msg.ParseFromString(msg.getDataAsString());
|
||||
auto message = std::make_shared<pb::DeleteMsg>(delete_msg);
|
||||
return std::shared_ptr<void>(message);
|
||||
}
|
||||
case SEARCH_RESULT: {
|
||||
pb::SearchResultMsg search_res_msg;
|
||||
search_res_msg.ParseFromString(msg.getDataAsString());
|
||||
auto message = std::make_shared<pb::SearchResultMsg>(search_res_msg);
|
||||
return std::shared_ptr<void>(message);
|
||||
}
|
||||
case TEST:
|
||||
pb::TestData test_msg;
|
||||
test_msg.ParseFromString(msg.getDataAsString());
|
||||
auto message = std::make_shared<pb::TestData>(test_msg);
|
||||
return std::shared_ptr<void>(message);
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
Result MsgConsumer::close() {
|
||||
return consumer_.close();
|
||||
}
|
||||
|
||||
Result MsgConsumer::acknowledge(const Message &message) {
|
||||
return consumer_.acknowledge(message);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
protoc -I=./ --cpp_out=./ pulsar.proto
|
||||
protoc -I=./ --cpp_out=./ suvlim.proto
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,143 +0,0 @@
|
|||
|
||||
syntax = "proto3";
|
||||
|
||||
package pb;
|
||||
|
||||
enum ErrorCode {
|
||||
SUCCESS = 0;
|
||||
UNEXPECTED_ERROR = 1;
|
||||
CONNECT_FAILED = 2;
|
||||
PERMISSION_DENIED = 3;
|
||||
COLLECTION_NOT_EXISTS = 4;
|
||||
ILLEGAL_ARGUMENT = 5;
|
||||
ILLEGAL_DIMENSION = 7;
|
||||
ILLEGAL_INDEX_TYPE = 8;
|
||||
ILLEGAL_COLLECTION_NAME = 9;
|
||||
ILLEGAL_TOPK = 10;
|
||||
ILLEGAL_ROWRECORD = 11;
|
||||
ILLEGAL_VECTOR_ID = 12;
|
||||
ILLEGAL_SEARCH_RESULT = 13;
|
||||
FILE_NOT_FOUND = 14;
|
||||
META_FAILED = 15;
|
||||
CACHE_FAILED = 16;
|
||||
CANNOT_CREATE_FOLDER = 17;
|
||||
CANNOT_CREATE_FILE = 18;
|
||||
CANNOT_DELETE_FOLDER = 19;
|
||||
CANNOT_DELETE_FILE = 20;
|
||||
BUILD_INDEX_ERROR = 21;
|
||||
ILLEGAL_NLIST = 22;
|
||||
ILLEGAL_METRIC_TYPE = 23;
|
||||
OUT_OF_MEMORY = 24;
|
||||
}
|
||||
|
||||
message Status {
|
||||
ErrorCode error_code = 1;
|
||||
string reason = 2;
|
||||
}
|
||||
|
||||
enum DataType {
|
||||
NONE = 0;
|
||||
BOOL = 1;
|
||||
INT8 = 2;
|
||||
INT16 = 3;
|
||||
INT32 = 4;
|
||||
INT64 = 5;
|
||||
|
||||
FLOAT = 10;
|
||||
DOUBLE = 11;
|
||||
|
||||
STRING = 20;
|
||||
|
||||
VECTOR_BINARY = 100;
|
||||
VECTOR_FLOAT = 101;
|
||||
}
|
||||
|
||||
enum OpType {
|
||||
Insert = 0;
|
||||
Delete = 1;
|
||||
Search = 2;
|
||||
TimeSync = 3;
|
||||
Key2Seg = 4;
|
||||
Statistics = 5;
|
||||
}
|
||||
|
||||
message SegmentRecord {
|
||||
repeated string seg_info = 1;
|
||||
}
|
||||
|
||||
message VectorRowRecord {
|
||||
repeated float float_data = 1; //float vector data
|
||||
bytes binary_data = 2; //binary vector data
|
||||
}
|
||||
|
||||
message AttrRecord {
|
||||
repeated int32 int32_value = 1;
|
||||
repeated int64 int64_value = 2;
|
||||
repeated float float_value = 3;
|
||||
repeated double double_value = 4;
|
||||
}
|
||||
|
||||
message VectorRecord {
|
||||
repeated VectorRowRecord records = 1;
|
||||
}
|
||||
|
||||
message VectorParam {
|
||||
string json = 1;
|
||||
VectorRecord row_record = 2;
|
||||
}
|
||||
|
||||
message FieldValue {
|
||||
string field_name = 1;
|
||||
DataType type = 2;
|
||||
AttrRecord attr_record = 3;
|
||||
VectorRecord vector_record = 4;
|
||||
}
|
||||
|
||||
message PulsarMessage {
|
||||
string collection_name = 1;
|
||||
repeated FieldValue fields = 2;
|
||||
int64 entity_id = 3;
|
||||
string partition_tag = 4;
|
||||
VectorParam vector_param =5;
|
||||
SegmentRecord segments = 6;
|
||||
int64 timestamp = 7;
|
||||
int64 client_id = 8;
|
||||
OpType msg_type = 9;
|
||||
string topic_name = 10;
|
||||
int64 partition_id = 11;
|
||||
}
|
||||
|
||||
//message PulsarMessages {
|
||||
// string collection_name = 1;
|
||||
// repeated FieldValue fields = 2;
|
||||
// repeated int64 entity_id = 3;
|
||||
// string partition_tag = 4;
|
||||
// repeated VectorParam vector_param =5;
|
||||
// repeated SegmentRecord segments = 6;
|
||||
// repeated int64 timestamp = 7;
|
||||
// repeated int64 client_id = 8;
|
||||
// OpType msg_type = 9;
|
||||
// repeated string topic_name = 10;
|
||||
// repeated int64 partition_id = 11;
|
||||
//}
|
||||
message TestData {
|
||||
string id = 1;
|
||||
string name = 2;
|
||||
}
|
||||
|
||||
message InsertMsg {
|
||||
int64 client_id = 1;
|
||||
}
|
||||
|
||||
message DeleteMsg {
|
||||
int64 client_id = 1;
|
||||
}
|
||||
|
||||
message SearchMsg {
|
||||
int64 client_id = 1;
|
||||
}
|
||||
|
||||
message SearchResultMsg {
|
||||
int64 client_id = 1;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,461 @@
|
|||
// Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
// source: status.proto
|
||||
|
||||
#include "status.pb.h"
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
#include <google/protobuf/stubs/common.h>
|
||||
#include <google/protobuf/io/coded_stream.h>
|
||||
#include <google/protobuf/extension_set.h>
|
||||
#include <google/protobuf/wire_format_lite.h>
|
||||
#include <google/protobuf/descriptor.h>
|
||||
#include <google/protobuf/generated_message_reflection.h>
|
||||
#include <google/protobuf/reflection_ops.h>
|
||||
#include <google/protobuf/wire_format.h>
|
||||
// @@protoc_insertion_point(includes)
|
||||
#include <google/protobuf/port_def.inc>
|
||||
namespace milvus {
|
||||
namespace grpc {
|
||||
class StatusDefaultTypeInternal {
|
||||
public:
|
||||
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<Status> _instance;
|
||||
} _Status_default_instance_;
|
||||
} // namespace grpc
|
||||
} // namespace milvus
|
||||
static void InitDefaultsscc_info_Status_status_2eproto() {
|
||||
GOOGLE_PROTOBUF_VERIFY_VERSION;
|
||||
|
||||
{
|
||||
void* ptr = &::milvus::grpc::_Status_default_instance_;
|
||||
new (ptr) ::milvus::grpc::Status();
|
||||
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
|
||||
}
|
||||
::milvus::grpc::Status::InitAsDefaultInstance();
|
||||
}
|
||||
|
||||
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_status_2eproto =
|
||||
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Status_status_2eproto}, {}};
|
||||
|
||||
static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_status_2eproto[1];
|
||||
static const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* file_level_enum_descriptors_status_2eproto[1];
|
||||
static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_status_2eproto = nullptr;
|
||||
|
||||
const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_status_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
|
||||
~0u, // no _has_bits_
|
||||
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, _internal_metadata_),
|
||||
~0u, // no _extensions_
|
||||
~0u, // no _oneof_case_
|
||||
~0u, // no _weak_field_map_
|
||||
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, error_code_),
|
||||
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Status, reason_),
|
||||
};
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
|
||||
{ 0, -1, sizeof(::milvus::grpc::Status)},
|
||||
};
|
||||
|
||||
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
|
||||
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::grpc::_Status_default_instance_),
|
||||
};
|
||||
|
||||
const char descriptor_table_protodef_status_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
|
||||
"\n\014status.proto\022\013milvus.grpc\"D\n\006Status\022*\n"
|
||||
"\nerror_code\030\001 \001(\0162\026.milvus.grpc.ErrorCod"
|
||||
"e\022\016\n\006reason\030\002 \001(\t*\242\004\n\tErrorCode\022\013\n\007SUCCE"
|
||||
"SS\020\000\022\024\n\020UNEXPECTED_ERROR\020\001\022\022\n\016CONNECT_FA"
|
||||
"ILED\020\002\022\025\n\021PERMISSION_DENIED\020\003\022\031\n\025COLLECT"
|
||||
"ION_NOT_EXISTS\020\004\022\024\n\020ILLEGAL_ARGUMENT\020\005\022\025"
|
||||
"\n\021ILLEGAL_DIMENSION\020\007\022\026\n\022ILLEGAL_INDEX_T"
|
||||
"YPE\020\010\022\033\n\027ILLEGAL_COLLECTION_NAME\020\t\022\020\n\014IL"
|
||||
"LEGAL_TOPK\020\n\022\025\n\021ILLEGAL_ROWRECORD\020\013\022\025\n\021I"
|
||||
"LLEGAL_VECTOR_ID\020\014\022\031\n\025ILLEGAL_SEARCH_RES"
|
||||
"ULT\020\r\022\022\n\016FILE_NOT_FOUND\020\016\022\017\n\013META_FAILED"
|
||||
"\020\017\022\020\n\014CACHE_FAILED\020\020\022\030\n\024CANNOT_CREATE_FO"
|
||||
"LDER\020\021\022\026\n\022CANNOT_CREATE_FILE\020\022\022\030\n\024CANNOT"
|
||||
"_DELETE_FOLDER\020\023\022\026\n\022CANNOT_DELETE_FILE\020\024"
|
||||
"\022\025\n\021BUILD_INDEX_ERROR\020\025\022\021\n\rILLEGAL_NLIST"
|
||||
"\020\026\022\027\n\023ILLEGAL_METRIC_TYPE\020\027\022\021\n\rOUT_OF_ME"
|
||||
"MORY\020\030b\006proto3"
|
||||
;
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_status_2eproto_deps[1] = {
|
||||
};
|
||||
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_status_2eproto_sccs[1] = {
|
||||
&scc_info_Status_status_2eproto.base,
|
||||
};
|
||||
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_status_2eproto_once;
|
||||
static bool descriptor_table_status_2eproto_initialized = false;
|
||||
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_status_2eproto = {
|
||||
&descriptor_table_status_2eproto_initialized, descriptor_table_protodef_status_2eproto, "status.proto", 654,
|
||||
&descriptor_table_status_2eproto_once, descriptor_table_status_2eproto_sccs, descriptor_table_status_2eproto_deps, 1, 0,
|
||||
schemas, file_default_instances, TableStruct_status_2eproto::offsets,
|
||||
file_level_metadata_status_2eproto, 1, file_level_enum_descriptors_status_2eproto, file_level_service_descriptors_status_2eproto,
|
||||
};
|
||||
|
||||
// Force running AddDescriptors() at dynamic initialization time.
|
||||
static bool dynamic_init_dummy_status_2eproto = ( ::PROTOBUF_NAMESPACE_ID::internal::AddDescriptors(&descriptor_table_status_2eproto), true);
|
||||
namespace milvus {
|
||||
namespace grpc {
|
||||
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ErrorCode_descriptor() {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_status_2eproto);
|
||||
return file_level_enum_descriptors_status_2eproto[0];
|
||||
}
|
||||
bool ErrorCode_IsValid(int value) {
|
||||
switch (value) {
|
||||
case 0:
|
||||
case 1:
|
||||
case 2:
|
||||
case 3:
|
||||
case 4:
|
||||
case 5:
|
||||
case 7:
|
||||
case 8:
|
||||
case 9:
|
||||
case 10:
|
||||
case 11:
|
||||
case 12:
|
||||
case 13:
|
||||
case 14:
|
||||
case 15:
|
||||
case 16:
|
||||
case 17:
|
||||
case 18:
|
||||
case 19:
|
||||
case 20:
|
||||
case 21:
|
||||
case 22:
|
||||
case 23:
|
||||
case 24:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ===================================================================
|
||||
|
||||
void Status::InitAsDefaultInstance() {
|
||||
}
|
||||
class Status::_Internal {
|
||||
public:
|
||||
};
|
||||
|
||||
Status::Status()
|
||||
: ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
|
||||
SharedCtor();
|
||||
// @@protoc_insertion_point(constructor:milvus.grpc.Status)
|
||||
}
|
||||
Status::Status(const Status& from)
|
||||
: ::PROTOBUF_NAMESPACE_ID::Message(),
|
||||
_internal_metadata_(nullptr) {
|
||||
_internal_metadata_.MergeFrom(from._internal_metadata_);
|
||||
reason_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
if (!from.reason().empty()) {
|
||||
reason_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.reason_);
|
||||
}
|
||||
error_code_ = from.error_code_;
|
||||
// @@protoc_insertion_point(copy_constructor:milvus.grpc.Status)
|
||||
}
|
||||
|
||||
void Status::SharedCtor() {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Status_status_2eproto.base);
|
||||
reason_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
error_code_ = 0;
|
||||
}
|
||||
|
||||
Status::~Status() {
|
||||
// @@protoc_insertion_point(destructor:milvus.grpc.Status)
|
||||
SharedDtor();
|
||||
}
|
||||
|
||||
void Status::SharedDtor() {
|
||||
reason_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
}
|
||||
|
||||
void Status::SetCachedSize(int size) const {
|
||||
_cached_size_.Set(size);
|
||||
}
|
||||
const Status& Status::default_instance() {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Status_status_2eproto.base);
|
||||
return *internal_default_instance();
|
||||
}
|
||||
|
||||
|
||||
void Status::Clear() {
|
||||
// @@protoc_insertion_point(message_clear_start:milvus.grpc.Status)
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
|
||||
// Prevent compiler warnings about cached_has_bits being unused
|
||||
(void) cached_has_bits;
|
||||
|
||||
reason_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
error_code_ = 0;
|
||||
_internal_metadata_.Clear();
|
||||
}
|
||||
|
||||
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
|
||||
const char* Status::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
|
||||
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
|
||||
while (!ctx->Done(&ptr)) {
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 tag;
|
||||
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
|
||||
CHK_(ptr);
|
||||
switch (tag >> 3) {
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
case 1:
|
||||
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 8)) {
|
||||
::PROTOBUF_NAMESPACE_ID::uint64 val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
|
||||
CHK_(ptr);
|
||||
set_error_code(static_cast<::milvus::grpc::ErrorCode>(val));
|
||||
} else goto handle_unusual;
|
||||
continue;
|
||||
// string reason = 2;
|
||||
case 2:
|
||||
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
|
||||
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_reason(), ptr, ctx, "milvus.grpc.Status.reason");
|
||||
CHK_(ptr);
|
||||
} else goto handle_unusual;
|
||||
continue;
|
||||
default: {
|
||||
handle_unusual:
|
||||
if ((tag & 7) == 4 || tag == 0) {
|
||||
ctx->SetLastTag(tag);
|
||||
goto success;
|
||||
}
|
||||
ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);
|
||||
CHK_(ptr != nullptr);
|
||||
continue;
|
||||
}
|
||||
} // switch
|
||||
} // while
|
||||
success:
|
||||
return ptr;
|
||||
failure:
|
||||
ptr = nullptr;
|
||||
goto success;
|
||||
#undef CHK_
|
||||
}
|
||||
#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
|
||||
bool Status::MergePartialFromCodedStream(
|
||||
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) {
|
||||
#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 tag;
|
||||
// @@protoc_insertion_point(parse_start:milvus.grpc.Status)
|
||||
for (;;) {
|
||||
::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
|
||||
tag = p.first;
|
||||
if (!p.second) goto handle_unusual;
|
||||
switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) {
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
case 1: {
|
||||
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (8 & 0xFF)) {
|
||||
int value = 0;
|
||||
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
|
||||
int, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_ENUM>(
|
||||
input, &value)));
|
||||
set_error_code(static_cast< ::milvus::grpc::ErrorCode >(value));
|
||||
} else {
|
||||
goto handle_unusual;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// string reason = 2;
|
||||
case 2: {
|
||||
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
|
||||
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString(
|
||||
input, this->mutable_reason()));
|
||||
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
|
||||
this->reason().data(), static_cast<int>(this->reason().length()),
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE,
|
||||
"milvus.grpc.Status.reason"));
|
||||
} else {
|
||||
goto handle_unusual;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default: {
|
||||
handle_unusual:
|
||||
if (tag == 0) {
|
||||
goto success;
|
||||
}
|
||||
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField(
|
||||
input, tag, _internal_metadata_.mutable_unknown_fields()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
success:
|
||||
// @@protoc_insertion_point(parse_success:milvus.grpc.Status)
|
||||
return true;
|
||||
failure:
|
||||
// @@protoc_insertion_point(parse_failure:milvus.grpc.Status)
|
||||
return false;
|
||||
#undef DO_
|
||||
}
|
||||
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
|
||||
|
||||
void Status::SerializeWithCachedSizes(
|
||||
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const {
|
||||
// @@protoc_insertion_point(serialize_start:milvus.grpc.Status)
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
|
||||
(void) cached_has_bits;
|
||||
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
if (this->error_code() != 0) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnum(
|
||||
1, this->error_code(), output);
|
||||
}
|
||||
|
||||
// string reason = 2;
|
||||
if (this->reason().size() > 0) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
|
||||
this->reason().data(), static_cast<int>(this->reason().length()),
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
|
||||
"milvus.grpc.Status.reason");
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased(
|
||||
2, this->reason(), output);
|
||||
}
|
||||
|
||||
if (_internal_metadata_.have_unknown_fields()) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
|
||||
_internal_metadata_.unknown_fields(), output);
|
||||
}
|
||||
// @@protoc_insertion_point(serialize_end:milvus.grpc.Status)
|
||||
}
|
||||
|
||||
::PROTOBUF_NAMESPACE_ID::uint8* Status::InternalSerializeWithCachedSizesToArray(
|
||||
::PROTOBUF_NAMESPACE_ID::uint8* target) const {
|
||||
// @@protoc_insertion_point(serialize_to_array_start:milvus.grpc.Status)
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
|
||||
(void) cached_has_bits;
|
||||
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
if (this->error_code() != 0) {
|
||||
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteEnumToArray(
|
||||
1, this->error_code(), target);
|
||||
}
|
||||
|
||||
// string reason = 2;
|
||||
if (this->reason().size() > 0) {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
|
||||
this->reason().data(), static_cast<int>(this->reason().length()),
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
|
||||
"milvus.grpc.Status.reason");
|
||||
target =
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray(
|
||||
2, this->reason(), target);
|
||||
}
|
||||
|
||||
if (_internal_metadata_.have_unknown_fields()) {
|
||||
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
|
||||
_internal_metadata_.unknown_fields(), target);
|
||||
}
|
||||
// @@protoc_insertion_point(serialize_to_array_end:milvus.grpc.Status)
|
||||
return target;
|
||||
}
|
||||
|
||||
size_t Status::ByteSizeLong() const {
|
||||
// @@protoc_insertion_point(message_byte_size_start:milvus.grpc.Status)
|
||||
size_t total_size = 0;
|
||||
|
||||
if (_internal_metadata_.have_unknown_fields()) {
|
||||
total_size +=
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize(
|
||||
_internal_metadata_.unknown_fields());
|
||||
}
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
|
||||
// Prevent compiler warnings about cached_has_bits being unused
|
||||
(void) cached_has_bits;
|
||||
|
||||
// string reason = 2;
|
||||
if (this->reason().size() > 0) {
|
||||
total_size += 1 +
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
|
||||
this->reason());
|
||||
}
|
||||
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
if (this->error_code() != 0) {
|
||||
total_size += 1 +
|
||||
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::EnumSize(this->error_code());
|
||||
}
|
||||
|
||||
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
|
||||
SetCachedSize(cached_size);
|
||||
return total_size;
|
||||
}
|
||||
|
||||
void Status::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
|
||||
// @@protoc_insertion_point(generalized_merge_from_start:milvus.grpc.Status)
|
||||
GOOGLE_DCHECK_NE(&from, this);
|
||||
const Status* source =
|
||||
::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated<Status>(
|
||||
&from);
|
||||
if (source == nullptr) {
|
||||
// @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.grpc.Status)
|
||||
::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);
|
||||
} else {
|
||||
// @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.grpc.Status)
|
||||
MergeFrom(*source);
|
||||
}
|
||||
}
|
||||
|
||||
void Status::MergeFrom(const Status& from) {
|
||||
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.grpc.Status)
|
||||
GOOGLE_DCHECK_NE(&from, this);
|
||||
_internal_metadata_.MergeFrom(from._internal_metadata_);
|
||||
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
|
||||
(void) cached_has_bits;
|
||||
|
||||
if (from.reason().size() > 0) {
|
||||
|
||||
reason_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.reason_);
|
||||
}
|
||||
if (from.error_code() != 0) {
|
||||
set_error_code(from.error_code());
|
||||
}
|
||||
}
|
||||
|
||||
void Status::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
|
||||
// @@protoc_insertion_point(generalized_copy_from_start:milvus.grpc.Status)
|
||||
if (&from == this) return;
|
||||
Clear();
|
||||
MergeFrom(from);
|
||||
}
|
||||
|
||||
void Status::CopyFrom(const Status& from) {
|
||||
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.grpc.Status)
|
||||
if (&from == this) return;
|
||||
Clear();
|
||||
MergeFrom(from);
|
||||
}
|
||||
|
||||
bool Status::IsInitialized() const {
|
||||
return true;
|
||||
}
|
||||
|
||||
void Status::InternalSwap(Status* other) {
|
||||
using std::swap;
|
||||
_internal_metadata_.Swap(&other->_internal_metadata_);
|
||||
reason_.Swap(&other->reason_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
|
||||
GetArenaNoVirtual());
|
||||
swap(error_code_, other->error_code_);
|
||||
}
|
||||
|
||||
::PROTOBUF_NAMESPACE_ID::Metadata Status::GetMetadata() const {
|
||||
return GetMetadataStatic();
|
||||
}
|
||||
|
||||
|
||||
// @@protoc_insertion_point(namespace_scope)
|
||||
} // namespace grpc
|
||||
} // namespace milvus
|
||||
PROTOBUF_NAMESPACE_OPEN
|
||||
template<> PROTOBUF_NOINLINE ::milvus::grpc::Status* Arena::CreateMaybeMessage< ::milvus::grpc::Status >(Arena* arena) {
|
||||
return Arena::CreateInternal< ::milvus::grpc::Status >(arena);
|
||||
}
|
||||
PROTOBUF_NAMESPACE_CLOSE
|
||||
|
||||
// @@protoc_insertion_point(global_scope)
|
||||
#include <google/protobuf/port_undef.inc>
|
||||
|
|
@ -0,0 +1,360 @@
|
|||
// Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
// source: status.proto
|
||||
|
||||
#ifndef GOOGLE_PROTOBUF_INCLUDED_status_2eproto
|
||||
#define GOOGLE_PROTOBUF_INCLUDED_status_2eproto
|
||||
|
||||
#include <limits>
|
||||
#include <string>
|
||||
|
||||
#include <google/protobuf/port_def.inc>
|
||||
#if PROTOBUF_VERSION < 3009000
|
||||
#error This file was generated by a newer version of protoc which is
|
||||
#error incompatible with your Protocol Buffer headers. Please update
|
||||
#error your headers.
|
||||
#endif
|
||||
#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION
|
||||
#error This file was generated by an older version of protoc which is
|
||||
#error incompatible with your Protocol Buffer headers. Please
|
||||
#error regenerate this file with a newer version of protoc.
|
||||
#endif
|
||||
|
||||
#include <google/protobuf/port_undef.inc>
|
||||
#include <google/protobuf/io/coded_stream.h>
|
||||
#include <google/protobuf/arena.h>
|
||||
#include <google/protobuf/arenastring.h>
|
||||
#include <google/protobuf/generated_message_table_driven.h>
|
||||
#include <google/protobuf/generated_message_util.h>
|
||||
#include <google/protobuf/inlined_string_field.h>
|
||||
#include <google/protobuf/metadata.h>
|
||||
#include <google/protobuf/generated_message_reflection.h>
|
||||
#include <google/protobuf/message.h>
|
||||
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
|
||||
#include <google/protobuf/extension_set.h> // IWYU pragma: export
|
||||
#include <google/protobuf/generated_enum_reflection.h>
|
||||
#include <google/protobuf/unknown_field_set.h>
|
||||
// @@protoc_insertion_point(includes)
|
||||
#include <google/protobuf/port_def.inc>
|
||||
#define PROTOBUF_INTERNAL_EXPORT_status_2eproto
|
||||
PROTOBUF_NAMESPACE_OPEN
|
||||
namespace internal {
|
||||
class AnyMetadata;
|
||||
} // namespace internal
|
||||
PROTOBUF_NAMESPACE_CLOSE
|
||||
|
||||
// Internal implementation detail -- do not use these members.
|
||||
struct TableStruct_status_2eproto {
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[]
|
||||
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
|
||||
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[1]
|
||||
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
|
||||
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
|
||||
static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[];
|
||||
};
|
||||
extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_status_2eproto;
|
||||
namespace milvus {
|
||||
namespace grpc {
|
||||
class Status;
|
||||
class StatusDefaultTypeInternal;
|
||||
extern StatusDefaultTypeInternal _Status_default_instance_;
|
||||
} // namespace grpc
|
||||
} // namespace milvus
|
||||
PROTOBUF_NAMESPACE_OPEN
|
||||
template<> ::milvus::grpc::Status* Arena::CreateMaybeMessage<::milvus::grpc::Status>(Arena*);
|
||||
PROTOBUF_NAMESPACE_CLOSE
|
||||
namespace milvus {
|
||||
namespace grpc {
|
||||
|
||||
enum ErrorCode : int {
|
||||
SUCCESS = 0,
|
||||
UNEXPECTED_ERROR = 1,
|
||||
CONNECT_FAILED = 2,
|
||||
PERMISSION_DENIED = 3,
|
||||
COLLECTION_NOT_EXISTS = 4,
|
||||
ILLEGAL_ARGUMENT = 5,
|
||||
ILLEGAL_DIMENSION = 7,
|
||||
ILLEGAL_INDEX_TYPE = 8,
|
||||
ILLEGAL_COLLECTION_NAME = 9,
|
||||
ILLEGAL_TOPK = 10,
|
||||
ILLEGAL_ROWRECORD = 11,
|
||||
ILLEGAL_VECTOR_ID = 12,
|
||||
ILLEGAL_SEARCH_RESULT = 13,
|
||||
FILE_NOT_FOUND = 14,
|
||||
META_FAILED = 15,
|
||||
CACHE_FAILED = 16,
|
||||
CANNOT_CREATE_FOLDER = 17,
|
||||
CANNOT_CREATE_FILE = 18,
|
||||
CANNOT_DELETE_FOLDER = 19,
|
||||
CANNOT_DELETE_FILE = 20,
|
||||
BUILD_INDEX_ERROR = 21,
|
||||
ILLEGAL_NLIST = 22,
|
||||
ILLEGAL_METRIC_TYPE = 23,
|
||||
OUT_OF_MEMORY = 24,
|
||||
ErrorCode_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
|
||||
ErrorCode_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max()
|
||||
};
|
||||
bool ErrorCode_IsValid(int value);
|
||||
constexpr ErrorCode ErrorCode_MIN = SUCCESS;
|
||||
constexpr ErrorCode ErrorCode_MAX = OUT_OF_MEMORY;
|
||||
constexpr int ErrorCode_ARRAYSIZE = ErrorCode_MAX + 1;
|
||||
|
||||
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* ErrorCode_descriptor();
|
||||
template<typename T>
|
||||
inline const std::string& ErrorCode_Name(T enum_t_value) {
|
||||
static_assert(::std::is_same<T, ErrorCode>::value ||
|
||||
::std::is_integral<T>::value,
|
||||
"Incorrect type passed to function ErrorCode_Name.");
|
||||
return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum(
|
||||
ErrorCode_descriptor(), enum_t_value);
|
||||
}
|
||||
inline bool ErrorCode_Parse(
|
||||
const std::string& name, ErrorCode* value) {
|
||||
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<ErrorCode>(
|
||||
ErrorCode_descriptor(), name, value);
|
||||
}
|
||||
// ===================================================================
|
||||
|
||||
class Status :
|
||||
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.grpc.Status) */ {
|
||||
public:
|
||||
Status();
|
||||
virtual ~Status();
|
||||
|
||||
Status(const Status& from);
|
||||
Status(Status&& from) noexcept
|
||||
: Status() {
|
||||
*this = ::std::move(from);
|
||||
}
|
||||
|
||||
inline Status& operator=(const Status& from) {
|
||||
CopyFrom(from);
|
||||
return *this;
|
||||
}
|
||||
inline Status& operator=(Status&& from) noexcept {
|
||||
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
|
||||
if (this != &from) InternalSwap(&from);
|
||||
} else {
|
||||
CopyFrom(from);
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
|
||||
return GetDescriptor();
|
||||
}
|
||||
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
|
||||
return GetMetadataStatic().descriptor;
|
||||
}
|
||||
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
|
||||
return GetMetadataStatic().reflection;
|
||||
}
|
||||
static const Status& default_instance();
|
||||
|
||||
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
|
||||
static inline const Status* internal_default_instance() {
|
||||
return reinterpret_cast<const Status*>(
|
||||
&_Status_default_instance_);
|
||||
}
|
||||
static constexpr int kIndexInFileMessages =
|
||||
0;
|
||||
|
||||
friend void swap(Status& a, Status& b) {
|
||||
a.Swap(&b);
|
||||
}
|
||||
inline void Swap(Status* other) {
|
||||
if (other == this) return;
|
||||
InternalSwap(other);
|
||||
}
|
||||
|
||||
// implements Message ----------------------------------------------
|
||||
|
||||
inline Status* New() const final {
|
||||
return CreateMaybeMessage<Status>(nullptr);
|
||||
}
|
||||
|
||||
Status* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
|
||||
return CreateMaybeMessage<Status>(arena);
|
||||
}
|
||||
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
|
||||
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
|
||||
void CopyFrom(const Status& from);
|
||||
void MergeFrom(const Status& from);
|
||||
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
|
||||
bool IsInitialized() const final;
|
||||
|
||||
size_t ByteSizeLong() const final;
|
||||
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
|
||||
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
|
||||
#else
|
||||
bool MergePartialFromCodedStream(
|
||||
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
|
||||
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
|
||||
void SerializeWithCachedSizes(
|
||||
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
|
||||
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
|
||||
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
|
||||
int GetCachedSize() const final { return _cached_size_.Get(); }
|
||||
|
||||
private:
|
||||
inline void SharedCtor();
|
||||
inline void SharedDtor();
|
||||
void SetCachedSize(int size) const final;
|
||||
void InternalSwap(Status* other);
|
||||
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
|
||||
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
|
||||
return "milvus.grpc.Status";
|
||||
}
|
||||
private:
|
||||
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
|
||||
return nullptr;
|
||||
}
|
||||
inline void* MaybeArenaPtr() const {
|
||||
return nullptr;
|
||||
}
|
||||
public:
|
||||
|
||||
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
|
||||
private:
|
||||
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
|
||||
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_status_2eproto);
|
||||
return ::descriptor_table_status_2eproto.file_level_metadata[kIndexInFileMessages];
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
// nested types ----------------------------------------------------
|
||||
|
||||
// accessors -------------------------------------------------------
|
||||
|
||||
enum : int {
|
||||
kReasonFieldNumber = 2,
|
||||
kErrorCodeFieldNumber = 1,
|
||||
};
|
||||
// string reason = 2;
|
||||
void clear_reason();
|
||||
const std::string& reason() const;
|
||||
void set_reason(const std::string& value);
|
||||
void set_reason(std::string&& value);
|
||||
void set_reason(const char* value);
|
||||
void set_reason(const char* value, size_t size);
|
||||
std::string* mutable_reason();
|
||||
std::string* release_reason();
|
||||
void set_allocated_reason(std::string* reason);
|
||||
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
void clear_error_code();
|
||||
::milvus::grpc::ErrorCode error_code() const;
|
||||
void set_error_code(::milvus::grpc::ErrorCode value);
|
||||
|
||||
// @@protoc_insertion_point(class_scope:milvus.grpc.Status)
|
||||
private:
|
||||
class _Internal;
|
||||
|
||||
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
|
||||
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr reason_;
|
||||
int error_code_;
|
||||
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
|
||||
friend struct ::TableStruct_status_2eproto;
|
||||
};
|
||||
// ===================================================================
|
||||
|
||||
|
||||
// ===================================================================
|
||||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
|
||||
#endif // __GNUC__
|
||||
// Status
|
||||
|
||||
// .milvus.grpc.ErrorCode error_code = 1;
|
||||
inline void Status::clear_error_code() {
|
||||
error_code_ = 0;
|
||||
}
|
||||
inline ::milvus::grpc::ErrorCode Status::error_code() const {
|
||||
// @@protoc_insertion_point(field_get:milvus.grpc.Status.error_code)
|
||||
return static_cast< ::milvus::grpc::ErrorCode >(error_code_);
|
||||
}
|
||||
inline void Status::set_error_code(::milvus::grpc::ErrorCode value) {
|
||||
|
||||
error_code_ = value;
|
||||
// @@protoc_insertion_point(field_set:milvus.grpc.Status.error_code)
|
||||
}
|
||||
|
||||
// string reason = 2;
|
||||
inline void Status::clear_reason() {
|
||||
reason_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
}
|
||||
inline const std::string& Status::reason() const {
|
||||
// @@protoc_insertion_point(field_get:milvus.grpc.Status.reason)
|
||||
return reason_.GetNoArena();
|
||||
}
|
||||
inline void Status::set_reason(const std::string& value) {
|
||||
|
||||
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
|
||||
// @@protoc_insertion_point(field_set:milvus.grpc.Status.reason)
|
||||
}
|
||||
inline void Status::set_reason(std::string&& value) {
|
||||
|
||||
reason_.SetNoArena(
|
||||
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
|
||||
// @@protoc_insertion_point(field_set_rvalue:milvus.grpc.Status.reason)
|
||||
}
|
||||
inline void Status::set_reason(const char* value) {
|
||||
GOOGLE_DCHECK(value != nullptr);
|
||||
|
||||
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
|
||||
// @@protoc_insertion_point(field_set_char:milvus.grpc.Status.reason)
|
||||
}
|
||||
inline void Status::set_reason(const char* value, size_t size) {
|
||||
|
||||
reason_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
|
||||
::std::string(reinterpret_cast<const char*>(value), size));
|
||||
// @@protoc_insertion_point(field_set_pointer:milvus.grpc.Status.reason)
|
||||
}
|
||||
inline std::string* Status::mutable_reason() {
|
||||
|
||||
// @@protoc_insertion_point(field_mutable:milvus.grpc.Status.reason)
|
||||
return reason_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
}
|
||||
inline std::string* Status::release_reason() {
|
||||
// @@protoc_insertion_point(field_release:milvus.grpc.Status.reason)
|
||||
|
||||
return reason_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
|
||||
}
|
||||
inline void Status::set_allocated_reason(std::string* reason) {
|
||||
if (reason != nullptr) {
|
||||
|
||||
} else {
|
||||
|
||||
}
|
||||
reason_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), reason);
|
||||
// @@protoc_insertion_point(field_set_allocated:milvus.grpc.Status.reason)
|
||||
}
|
||||
|
||||
#ifdef __GNUC__
|
||||
#pragma GCC diagnostic pop
|
||||
#endif // __GNUC__
|
||||
|
||||
// @@protoc_insertion_point(namespace_scope)
|
||||
|
||||
} // namespace grpc
|
||||
} // namespace milvus
|
||||
|
||||
PROTOBUF_NAMESPACE_OPEN
|
||||
|
||||
template <> struct is_proto_enum< ::milvus::grpc::ErrorCode> : ::std::true_type {};
|
||||
template <>
|
||||
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::grpc::ErrorCode>() {
|
||||
return ::milvus::grpc::ErrorCode_descriptor();
|
||||
}
|
||||
|
||||
PROTOBUF_NAMESPACE_CLOSE
|
||||
|
||||
// @@protoc_insertion_point(global_scope)
|
||||
|
||||
#include <google/protobuf/port_undef.inc>
|
||||
#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_status_2eproto
|
||||
|
|
@ -0,0 +1,35 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package milvus.grpc;
|
||||
|
||||
enum ErrorCode {
|
||||
SUCCESS = 0;
|
||||
UNEXPECTED_ERROR = 1;
|
||||
CONNECT_FAILED = 2;
|
||||
PERMISSION_DENIED = 3;
|
||||
COLLECTION_NOT_EXISTS = 4;
|
||||
ILLEGAL_ARGUMENT = 5;
|
||||
ILLEGAL_DIMENSION = 7;
|
||||
ILLEGAL_INDEX_TYPE = 8;
|
||||
ILLEGAL_COLLECTION_NAME = 9;
|
||||
ILLEGAL_TOPK = 10;
|
||||
ILLEGAL_ROWRECORD = 11;
|
||||
ILLEGAL_VECTOR_ID = 12;
|
||||
ILLEGAL_SEARCH_RESULT = 13;
|
||||
FILE_NOT_FOUND = 14;
|
||||
META_FAILED = 15;
|
||||
CACHE_FAILED = 16;
|
||||
CANNOT_CREATE_FOLDER = 17;
|
||||
CANNOT_CREATE_FILE = 18;
|
||||
CANNOT_DELETE_FOLDER = 19;
|
||||
CANNOT_DELETE_FILE = 20;
|
||||
BUILD_INDEX_ERROR = 21;
|
||||
ILLEGAL_NLIST = 22;
|
||||
ILLEGAL_METRIC_TYPE = 23;
|
||||
OUT_OF_MEMORY = 24;
|
||||
}
|
||||
|
||||
message Status {
|
||||
ErrorCode error_code = 1;
|
||||
string reason = 2;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
|
@ -1,38 +1,40 @@
|
|||
syntax = "proto3";
|
||||
|
||||
package pb;
|
||||
import "status.proto";
|
||||
package milvus.grpc;
|
||||
|
||||
enum ErrorCode {
|
||||
SUCCESS = 0;
|
||||
UNEXPECTED_ERROR = 1;
|
||||
CONNECT_FAILED = 2;
|
||||
PERMISSION_DENIED = 3;
|
||||
COLLECTION_NOT_EXISTS = 4;
|
||||
ILLEGAL_ARGUMENT = 5;
|
||||
ILLEGAL_DIMENSION = 7;
|
||||
ILLEGAL_INDEX_TYPE = 8;
|
||||
ILLEGAL_COLLECTION_NAME = 9;
|
||||
ILLEGAL_TOPK = 10;
|
||||
ILLEGAL_ROWRECORD = 11;
|
||||
ILLEGAL_VECTOR_ID = 12;
|
||||
ILLEGAL_SEARCH_RESULT = 13;
|
||||
FILE_NOT_FOUND = 14;
|
||||
META_FAILED = 15;
|
||||
CACHE_FAILED = 16;
|
||||
CANNOT_CREATE_FOLDER = 17;
|
||||
CANNOT_CREATE_FILE = 18;
|
||||
CANNOT_DELETE_FOLDER = 19;
|
||||
CANNOT_DELETE_FILE = 20;
|
||||
BUILD_INDEX_ERROR = 21;
|
||||
ILLEGAL_NLIST = 22;
|
||||
ILLEGAL_METRIC_TYPE = 23;
|
||||
OUT_OF_MEMORY = 24;
|
||||
}
|
||||
|
||||
message Status {
|
||||
ErrorCode error_code = 1;
|
||||
string reason = 2;
|
||||
}
|
||||
//
|
||||
//enum ErrorCode {
|
||||
// SUCCESS = 0;
|
||||
// UNEXPECTED_ERROR = 1;
|
||||
// CONNECT_FAILED = 2;
|
||||
// PERMISSION_DENIED = 3;
|
||||
// COLLECTION_NOT_EXISTS = 4;
|
||||
// ILLEGAL_ARGUMENT = 5;
|
||||
// ILLEGAL_DIMENSION = 7;
|
||||
// ILLEGAL_INDEX_TYPE = 8;
|
||||
// ILLEGAL_COLLECTION_NAME = 9;
|
||||
// ILLEGAL_TOPK = 10;
|
||||
// ILLEGAL_ROWRECORD = 11;
|
||||
// ILLEGAL_VECTOR_ID = 12;
|
||||
// ILLEGAL_SEARCH_RESULT = 13;
|
||||
// FILE_NOT_FOUND = 14;
|
||||
// META_FAILED = 15;
|
||||
// CACHE_FAILED = 16;
|
||||
// CANNOT_CREATE_FOLDER = 17;
|
||||
// CANNOT_CREATE_FILE = 18;
|
||||
// CANNOT_DELETE_FOLDER = 19;
|
||||
// CANNOT_DELETE_FILE = 20;
|
||||
// BUILD_INDEX_ERROR = 21;
|
||||
// ILLEGAL_NLIST = 22;
|
||||
// ILLEGAL_METRIC_TYPE = 23;
|
||||
// OUT_OF_MEMORY = 24;
|
||||
//}
|
||||
//
|
||||
//message Status {
|
||||
// ErrorCode error_code = 1;
|
||||
// string reason = 2;
|
||||
//}
|
||||
|
||||
/**
|
||||
* @brief Field data type
|
||||
|
|
@ -97,7 +99,7 @@ message FieldName {
|
|||
message Mapping {
|
||||
Status status = 1;
|
||||
string collection_name = 2;
|
||||
repeated FieldParam fields = 3;
|
||||
Schema schema = 3;
|
||||
repeated KeyValuePair extra_params = 4;
|
||||
}
|
||||
|
||||
|
|
@ -133,147 +135,70 @@ message VectorRowRecord {
|
|||
bytes binary_data = 2; //binary vector data
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Attribute record
|
||||
*/
|
||||
message AttrRecord {
|
||||
repeated int32 int32_value = 1;
|
||||
repeated int64 int64_value = 2;
|
||||
repeated float float_value = 3;
|
||||
repeated double double_value = 4;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Vector records
|
||||
*/
|
||||
message VectorRecord {
|
||||
repeated VectorRowRecord records = 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Field values
|
||||
*/
|
||||
message FieldValue {
|
||||
string field_name = 1;
|
||||
DataType type = 2;
|
||||
AttrRecord attr_record = 3;
|
||||
VectorRecord vector_record = 4;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Parameters for insert action
|
||||
*/
|
||||
message InsertParam {
|
||||
string collection_name = 1;
|
||||
repeated FieldValue fields = 2;
|
||||
repeated int64 entity_id_array = 3; //optional
|
||||
string partition_tag = 4;
|
||||
repeated KeyValuePair extra_params = 5;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Entity ids
|
||||
*/
|
||||
message EntityIds {
|
||||
Status status = 1;
|
||||
repeated int64 entity_id_array = 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Search vector parameters
|
||||
*/
|
||||
message VectorRecord {
|
||||
repeated VectorRowRecord records = 1;
|
||||
}
|
||||
|
||||
message VectorParam {
|
||||
string json = 1;
|
||||
VectorRecord row_record = 2;
|
||||
}
|
||||
|
||||
//////////////////////////row schema and data///////////////////////////////////
|
||||
/**
|
||||
* @brief Parameters for search action
|
||||
* @dsl example:
|
||||
* {
|
||||
* "query": {
|
||||
* "bool": {
|
||||
* "must": [
|
||||
* {
|
||||
* "must":[
|
||||
* {
|
||||
* "should": [
|
||||
* {
|
||||
* "term": {
|
||||
* "gender": ["male"]
|
||||
* }
|
||||
* },
|
||||
* {
|
||||
* "range": {
|
||||
* "height": {"gte": "170.0", "lte": "180.0"}
|
||||
* }
|
||||
* }
|
||||
* ]
|
||||
* },
|
||||
* {
|
||||
* "must_not": [
|
||||
* {
|
||||
* "term": {
|
||||
* "age": [20, 21, 22, 23, 24, 25]
|
||||
* }
|
||||
* },
|
||||
* {
|
||||
* "Range": {
|
||||
* "weight": {"lte": "100"}
|
||||
* }
|
||||
* }
|
||||
* ]
|
||||
* }
|
||||
* ]
|
||||
* },
|
||||
* {
|
||||
* "must": [
|
||||
* {
|
||||
* "vector": {
|
||||
* "face_img": {
|
||||
* "topk": 10,
|
||||
* "metric_type": "L2",
|
||||
* "query": [],
|
||||
* "params": {
|
||||
* "nprobe": 10
|
||||
* }
|
||||
* }
|
||||
* }
|
||||
* }
|
||||
* ]
|
||||
* }
|
||||
* ]
|
||||
* }
|
||||
* },
|
||||
* "fields": ["age", "face_img"]
|
||||
* }
|
||||
* @brief schema
|
||||
*/
|
||||
message FieldMeta {
|
||||
string field_name = 1;
|
||||
DataType type = 2;
|
||||
int64 dim = 3;
|
||||
}
|
||||
|
||||
message Schema {
|
||||
repeated FieldMeta field_metas = 1;
|
||||
}
|
||||
|
||||
message RowData {
|
||||
bytes blob = 1;
|
||||
}
|
||||
|
||||
//////////////////////suvlim-proxy///////////////////////////////////
|
||||
message InsertParam {
|
||||
string collection_name = 1;
|
||||
Schema schema = 2;
|
||||
repeated RowData rows_data = 3;
|
||||
repeated int64 entity_id_array = 4; //optional
|
||||
string partition_tag = 5;
|
||||
repeated KeyValuePair extra_params = 6;
|
||||
}
|
||||
|
||||
message SearchParam {
|
||||
string collection_name = 1;
|
||||
repeated string partition_tag_array = 2;
|
||||
repeated VectorParam vector_param = 3;
|
||||
string dsl = 4;
|
||||
repeated VectorParam vector_param = 2;
|
||||
string dsl = 3; //optional
|
||||
repeated string partition_tag = 4; //why
|
||||
repeated KeyValuePair extra_params = 5;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Parameters for searching in segments
|
||||
*/
|
||||
message SearchInSegmentParam {
|
||||
repeated string file_id_array = 1;
|
||||
SearchParam search_param = 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Entities
|
||||
*/
|
||||
message Entities {
|
||||
Status status = 1;
|
||||
repeated int64 ids = 2;
|
||||
repeated bool valid_row = 3;
|
||||
repeated FieldValue fields = 4;
|
||||
repeated RowData rows_data = 4;
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////milvus-server///////////////////////////
|
||||
/**
|
||||
* @brief Query result
|
||||
*/
|
||||
|
|
@ -743,92 +668,51 @@ service MilvusService {
|
|||
rpc SearchPB(SearchParamPB) returns (QueryResult) {}
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////
|
||||
////////////////////pulsar//////////////////////////////////////
|
||||
enum OpType {
|
||||
Insert = 0;
|
||||
Delete = 1;
|
||||
Search = 2;
|
||||
TimeSync = 3;
|
||||
Key2Seg = 4;
|
||||
Statistics = 5;
|
||||
INSERT = 0;
|
||||
DELETE = 1;
|
||||
}
|
||||
|
||||
message Key2SegmentRecord {
|
||||
repeated uint64 segment_id = 1;
|
||||
}
|
||||
|
||||
message AttrRowRecord {
|
||||
int32 int32_value = 1;
|
||||
int64 int64_value = 2;
|
||||
float float_value = 3;
|
||||
double double_value = 4;
|
||||
}
|
||||
|
||||
message VectorRowParam {
|
||||
string json = 1;
|
||||
VectorRowRecord row_record = 2;
|
||||
}
|
||||
|
||||
message FieldRowValue {
|
||||
string field_name = 1;
|
||||
DataType type = 2;
|
||||
AttrRowRecord attr_record = 3;
|
||||
VectorRowRecord vector_record = 4;
|
||||
}
|
||||
|
||||
message PulsarMessage {
|
||||
message InsertOrDeleteMsg {
|
||||
string collection_name = 1;
|
||||
repeated FieldRowValue fields = 2;
|
||||
uint64 entity_id = 3;
|
||||
RowData rows_data = 2;
|
||||
int64 uid = 3; //optional
|
||||
string partition_tag = 4;
|
||||
VectorRowParam vector_param =5;
|
||||
Key2SegmentRecord segments = 6;
|
||||
int64 timestamp = 7;
|
||||
int64 client_id = 8;
|
||||
OpType msg_type = 9;
|
||||
string topic_name = 10;
|
||||
int64 partition_id = 11;
|
||||
uint64 segment_id = 12;
|
||||
}
|
||||
|
||||
message InsertMsg {
|
||||
string collection_name = 1;
|
||||
repeated FieldRowValue fields = 2;
|
||||
uint64 entity_id = 3;
|
||||
string partition_tag = 4;
|
||||
int64 timestamp = 5;
|
||||
int64 client_id = 6;
|
||||
OpType msg_type = 7;
|
||||
string topic_name = 8;
|
||||
int64 partition_id = 9;
|
||||
uint64 segment_id = 10;
|
||||
}
|
||||
|
||||
message DeleteMsg {
|
||||
string collection_name = 1;
|
||||
uint64 entity_id = 3;
|
||||
int64 timestamp = 5;
|
||||
int64 client_id = 6;
|
||||
OpType msg_type = 7;
|
||||
uint64 timestamp =5;
|
||||
int64 segment_id = 6;
|
||||
int64 channel_id = 7;
|
||||
OpType op = 8;
|
||||
int64 client_id = 9;
|
||||
repeated KeyValuePair extra_params = 10;
|
||||
}
|
||||
|
||||
message SearchMsg {
|
||||
string collection_name = 1;
|
||||
string partition_tag = 2;
|
||||
VectorRowParam vector_param = 3;
|
||||
int64 timestamp = 4;
|
||||
int64 client_id = 5;
|
||||
OpType msg_type = 6;
|
||||
VectorRowRecord records = 2;
|
||||
string partition_tag = 3;
|
||||
int64 uid = 4;
|
||||
uint64 timestamp =5;
|
||||
int64 client_id = 6;
|
||||
repeated KeyValuePair extra_params = 7;
|
||||
}
|
||||
|
||||
message TimeSyncMsg {
|
||||
int64 timestamp = 1;
|
||||
int64 client_id = 2;
|
||||
OpType msg_type = 3;
|
||||
}
|
||||
enum SyncType {
|
||||
READ = 0;
|
||||
WRITE = 1;
|
||||
}
|
||||
message TimeSyncMsg{
|
||||
int64 peer_Id = 1;
|
||||
uint64 Timestamp = 2;
|
||||
SyncType sync_type = 3;
|
||||
}
|
||||
|
||||
message SegmentRecord {
|
||||
int64 uid = 1;
|
||||
repeated int64 segment_id = 2;
|
||||
}
|
||||
|
||||
message Key2SegMsg {
|
||||
int64 client_id = 1;
|
||||
Key2SegmentRecord segments = 2;
|
||||
OpType msg_type = 3;
|
||||
}
|
||||
SegmentRecord records = 2;
|
||||
}
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
|
||||
#include "producer.h"
|
||||
|
||||
namespace message_client {
|
||||
|
||||
MsgProducer::MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic, const ProducerConfiguration conf) : client_(client), config_(conf){
|
||||
createProducer(topic);
|
||||
}
|
||||
|
||||
Result MsgProducer::createProducer(const std::string &topic) {
|
||||
return client_->createProducer(topic, producer_);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const Message &msg) {
|
||||
return producer_.send(msg);
|
||||
}
|
||||
|
||||
Result MsgProducer::send(const std::string &msg) {
|
||||
auto pulsar_msg = pulsar::MessageBuilder().setContent(msg).build();
|
||||
return send(pulsar_msg);
|
||||
}
|
||||
|
||||
Result MsgProducer::close() {
|
||||
return producer_.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "pulsar/Producer.h"
|
||||
#include "client.h"
|
||||
|
||||
namespace message_client {
|
||||
|
||||
using Producer = pulsar::Producer;
|
||||
using ProducerConfiguration = pulsar::ProducerConfiguration;
|
||||
|
||||
class MsgProducer{
|
||||
public:
|
||||
MsgProducer(std::shared_ptr<MsgClient> &client, const std::string &topic, const ProducerConfiguration conf = ProducerConfiguration());
|
||||
|
||||
Result createProducer(const std::string& topic);
|
||||
Result send(const Message& msg);
|
||||
Result send(const std::string& msg);
|
||||
Result close();
|
||||
|
||||
const Producer&
|
||||
producer() const { return producer_; }
|
||||
|
||||
private:
|
||||
Producer producer_;
|
||||
std::shared_ptr<MsgClient> client_;
|
||||
ProducerConfiguration config_;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -15,7 +15,7 @@ type minioDriver struct {
|
|||
|
||||
func NewMinioDriver(ctx context.Context) (*minioDriver, error) {
|
||||
// to-do read conf
|
||||
var endPoint = "127.0.0.1:9000"
|
||||
var endPoint = "192.168.2.28:9000"
|
||||
var accessKeyID = "testminio"
|
||||
var secretAccessKey = "testminio"
|
||||
var useSSL = false
|
||||
|
|
|
|||
|
|
@ -3,51 +3,33 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/czs007/suvlim/storage/pkg"
|
||||
"github.com/czs007/suvlim/storage/pkg/types"
|
||||
"github.com/czs007/suvlim/writer/message_client"
|
||||
"github.com/czs007/suvlim/writer/write_node"
|
||||
"log"
|
||||
"sync"
|
||||
"time"
|
||||
"writer/message_client"
|
||||
"writer/mock"
|
||||
"writer/pb"
|
||||
"writer/write_node"
|
||||
)
|
||||
|
||||
func GetInsertMsg(collectionName string, partitionTag string, entityId int64) *pb.InsertOrDeleteMsg {
|
||||
return &pb.InsertOrDeleteMsg{
|
||||
CollectionName: collectionName,
|
||||
PartitionTag: partitionTag,
|
||||
SegmentId: int64(entityId / 100),
|
||||
Uid: int64(entityId),
|
||||
Timestamp: int64(entityId),
|
||||
ClientId: 0,
|
||||
}
|
||||
}
|
||||
|
||||
func GetDeleteMsg(collectionName string, entityId int64) *pb.InsertOrDeleteMsg {
|
||||
return &pb.InsertOrDeleteMsg{
|
||||
CollectionName: collectionName,
|
||||
Uid: entityId,
|
||||
Timestamp: int64(entityId + 100),
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
mc := message_client.MessageClient{}
|
||||
mc.InitClient("pulsar://localhost:6650")
|
||||
//mc.InitClient("pulsar://192.168.2.18:6650")
|
||||
//TODO::close client / consumer/ producer
|
||||
//mc.Close()
|
||||
|
||||
go mc.ReceiveMessage()
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
kv, err := mock.NewTikvStore()
|
||||
kv, err := storage.NewStore(context.Background(), types.MinIODriver)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
wn := write_node.WriteNode{
|
||||
KvStore: kv,
|
||||
KvStore: &kv,
|
||||
MessageClient: &mc,
|
||||
TimeSync: 100,
|
||||
}
|
||||
|
|
@ -56,12 +38,13 @@ func main() {
|
|||
for {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
msgLength := wn.MessageClient.PrepareBatchMsg()
|
||||
readyDo := true
|
||||
readyDo := false
|
||||
for _, len := range msgLength {
|
||||
if len <= 0 { readyDo = false }
|
||||
if len > 0 { readyDo = true }
|
||||
}
|
||||
if readyDo {
|
||||
wn.DoWriteNode(ctx, 100, wg)
|
||||
wn.DoWriteNode(ctx, 100, &wg)
|
||||
fmt.Println("write node do a batch message, storage len: ")
|
||||
}
|
||||
fmt.Println("do a batch in 200ms")
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,35 +3,34 @@ package message_client
|
|||
import (
|
||||
"context"
|
||||
"github.com/apache/pulsar/pulsar-client-go/pulsar"
|
||||
"github.com/czs007/suvlim/writer/pb"
|
||||
"github.com/golang/protobuf/proto"
|
||||
"log"
|
||||
"writer/pb"
|
||||
)
|
||||
|
||||
|
||||
type MessageClient struct {
|
||||
|
||||
//message channel
|
||||
insertOrDeleteChan chan *pb.InsertOrDeleteMsg
|
||||
searchChan chan *pb.SearchMsg
|
||||
timeSyncChan chan *pb.TimeSyncMsg
|
||||
searchByIdChan chan *pb.EntityIdentity
|
||||
timeSyncChan chan *pb.TimeSyncMsg
|
||||
|
||||
// pulsar
|
||||
client pulsar.Client
|
||||
key2segProducer pulsar.Producer
|
||||
writeSyncProducer pulsar.Producer
|
||||
client pulsar.Client
|
||||
key2segProducer pulsar.Producer
|
||||
writeSyncProducer pulsar.Producer
|
||||
insertOrDeleteConsumer pulsar.Consumer
|
||||
searchConsumer pulsar.Consumer
|
||||
timeSyncConsumer pulsar.Consumer
|
||||
searchByIdConsumer pulsar.Consumer
|
||||
timeSyncConsumer pulsar.Consumer
|
||||
|
||||
// batch messages
|
||||
InsertMsg []*pb.InsertOrDeleteMsg
|
||||
DeleteMsg []*pb.InsertOrDeleteMsg
|
||||
SearchMsg []*pb.SearchMsg
|
||||
timeSyncMsg []*pb.TimeSyncMsg
|
||||
InsertMsg []*pb.InsertOrDeleteMsg
|
||||
DeleteMsg []*pb.InsertOrDeleteMsg
|
||||
SearchByIdMsg []*pb.EntityIdentity
|
||||
timeSyncMsg []*pb.TimeSyncMsg
|
||||
}
|
||||
|
||||
func (mc *MessageClient)ReceiveInsertOrDeleteMsg() {
|
||||
func (mc *MessageClient) ReceiveInsertOrDeleteMsg() {
|
||||
for {
|
||||
insetOrDeleteMsg := pb.InsertOrDeleteMsg{}
|
||||
msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
|
||||
|
|
@ -40,40 +39,43 @@ func (mc *MessageClient)ReceiveInsertOrDeleteMsg() {
|
|||
log.Fatal(err)
|
||||
}
|
||||
mc.insertOrDeleteChan <- &insetOrDeleteMsg
|
||||
mc.insertOrDeleteConsumer.Ack(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MessageClient)ReceiveSearchMsg() {
|
||||
func (mc *MessageClient) ReceiveSearchByIdMsg() {
|
||||
for {
|
||||
searchMsg := pb.SearchMsg{}
|
||||
msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
|
||||
err = proto.Unmarshal(msg.Payload(), &searchMsg)
|
||||
searchByIdMsg := pb.EntityIdentity{}
|
||||
msg, err := mc.searchByIdConsumer.Receive(context.Background())
|
||||
err = proto.Unmarshal(msg.Payload(), &searchByIdMsg)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
mc.searchChan <- &searchMsg
|
||||
mc.searchByIdChan <- &searchByIdMsg
|
||||
mc.searchByIdConsumer.Ack(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MessageClient)ReceiveTimeSyncMsg() {
|
||||
func (mc *MessageClient) ReceiveTimeSyncMsg() {
|
||||
for {
|
||||
timeSyncMsg := pb.TimeSyncMsg{}
|
||||
msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
|
||||
msg, err := mc.timeSyncConsumer.Receive(context.Background())
|
||||
err = proto.Unmarshal(msg.Payload(), &timeSyncMsg)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
mc.timeSyncChan <- &timeSyncMsg
|
||||
mc.timeSyncConsumer.Ack(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MessageClient) ReceiveMessage() {
|
||||
go mc.ReceiveInsertOrDeleteMsg()
|
||||
go mc.ReceiveSearchMsg()
|
||||
go mc.ReceiveSearchByIdMsg()
|
||||
go mc.ReceiveTimeSyncMsg()
|
||||
}
|
||||
|
||||
func (mc *MessageClient) CreatProducer(topicName string) pulsar.Producer{
|
||||
func (mc *MessageClient) CreatProducer(topicName string) pulsar.Producer {
|
||||
producer, err := mc.client.CreateProducer(pulsar.ProducerOptions{
|
||||
Topic: topicName,
|
||||
})
|
||||
|
|
@ -86,7 +88,7 @@ func (mc *MessageClient) CreatProducer(topicName string) pulsar.Producer{
|
|||
|
||||
func (mc *MessageClient) CreateConsumer(topicName string) pulsar.Consumer {
|
||||
consumer, err := mc.client.Subscribe(pulsar.ConsumerOptions{
|
||||
Topic: topicName,
|
||||
Topic: topicName,
|
||||
SubscriptionName: "multi-topic-sub",
|
||||
})
|
||||
|
||||
|
|
@ -118,13 +120,18 @@ func (mc *MessageClient) InitClient(url string) {
|
|||
|
||||
//create consumer
|
||||
mc.insertOrDeleteConsumer = mc.CreateConsumer("InsertOrDelete")
|
||||
mc.searchConsumer = mc.CreateConsumer("Search")
|
||||
mc.searchByIdConsumer = mc.CreateConsumer("SearchById")
|
||||
mc.timeSyncConsumer = mc.CreateConsumer("TimeSync")
|
||||
|
||||
// init channel
|
||||
mc.insertOrDeleteChan = make(chan *pb.InsertOrDeleteMsg, 1000)
|
||||
mc.searchChan = make(chan *pb.SearchMsg, 1000)
|
||||
mc.searchByIdChan = make(chan *pb.EntityIdentity, 1000)
|
||||
mc.timeSyncChan = make(chan *pb.TimeSyncMsg, 1000)
|
||||
|
||||
mc.InsertMsg = make([]*pb.InsertOrDeleteMsg, 1000)
|
||||
mc.DeleteMsg = make([]*pb.InsertOrDeleteMsg, 1000)
|
||||
mc.SearchByIdMsg = make([]*pb.EntityIdentity, 1000)
|
||||
mc.timeSyncMsg = make([]*pb.TimeSyncMsg, 1000)
|
||||
}
|
||||
|
||||
func (mc *MessageClient) Close() {
|
||||
|
|
@ -132,21 +139,23 @@ func (mc *MessageClient) Close() {
|
|||
defer mc.key2segProducer.Close()
|
||||
defer mc.writeSyncProducer.Close()
|
||||
defer mc.insertOrDeleteConsumer.Close()
|
||||
defer mc.searchConsumer.Close()
|
||||
defer mc.searchByIdConsumer.Close()
|
||||
defer mc.timeSyncConsumer.Close()
|
||||
}
|
||||
|
||||
type JobType int
|
||||
|
||||
const (
|
||||
OpInQueryNode JobType = 0
|
||||
OpInWriteNode JobType = 1
|
||||
)
|
||||
|
||||
type MessageType int
|
||||
|
||||
const (
|
||||
InsertOrDelete MessageType = 0
|
||||
Delete MessageType = 1
|
||||
Search MessageType = 2
|
||||
SearchById MessageType = 2
|
||||
TimeSync MessageType = 3
|
||||
Key2Seg MessageType = 4
|
||||
Statistics MessageType = 5
|
||||
|
|
@ -158,40 +167,40 @@ func (mc *MessageClient) PrepareMsg(messageType MessageType, msgLen int) {
|
|||
for i := 0; i < msgLen; i++ {
|
||||
msg := <-mc.insertOrDeleteChan
|
||||
if msg.Op == pb.OpType_INSERT {
|
||||
mc.InsertMsg[i] = msg
|
||||
mc.InsertMsg = append(mc.InsertMsg, msg)
|
||||
} else {
|
||||
mc.DeleteMsg[i] = msg
|
||||
mc.DeleteMsg = append(mc.DeleteMsg, msg)
|
||||
}
|
||||
}
|
||||
case Search:
|
||||
case SearchById:
|
||||
for i := 0; i < msgLen; i++ {
|
||||
msg := <-mc.searchChan
|
||||
mc.SearchMsg[i] = msg
|
||||
msg := <-mc.searchByIdChan
|
||||
mc.SearchByIdMsg = append(mc.SearchByIdMsg, msg)
|
||||
}
|
||||
case TimeSync:
|
||||
for i := 0; i < msgLen; i++ {
|
||||
msg := <-mc.timeSyncChan
|
||||
mc.timeSyncMsg[i] = msg
|
||||
mc.timeSyncMsg = append(mc.timeSyncMsg, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (mc *MessageClient)PrepareBatchMsg() []int{
|
||||
func (mc *MessageClient) PrepareBatchMsg() []int {
|
||||
// assume the channel not full
|
||||
mc.InsertMsg = make([]*pb.InsertOrDeleteMsg, 1000)
|
||||
mc.DeleteMsg = make([]*pb.InsertOrDeleteMsg, 1000)
|
||||
mc.SearchMsg = make([]*pb.SearchMsg, 1000)
|
||||
mc.timeSyncMsg = make([]*pb.TimeSyncMsg, 1000)
|
||||
mc.InsertMsg = mc.InsertMsg[:0]
|
||||
mc.DeleteMsg = mc.DeleteMsg[:0]
|
||||
mc.SearchByIdMsg = mc.SearchByIdMsg[:0]
|
||||
mc.timeSyncMsg = mc.timeSyncMsg[:0]
|
||||
|
||||
// get the length of every channel
|
||||
insertOrDeleteLen := len(mc.insertOrDeleteChan)
|
||||
searchLen := len(mc.searchChan)
|
||||
searchLen := len(mc.searchByIdChan)
|
||||
timeLen := len(mc.timeSyncChan)
|
||||
|
||||
// get message from channel to slice
|
||||
mc.PrepareMsg(InsertOrDelete, insertOrDeleteLen)
|
||||
mc.PrepareMsg(Search, searchLen)
|
||||
mc.PrepareMsg(SearchById, searchLen)
|
||||
mc.PrepareMsg(TimeSync, timeLen)
|
||||
|
||||
return []int{insertOrDeleteLen, searchLen, timeLen}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ package test
|
|||
|
||||
import (
|
||||
"context"
|
||||
"github.com/czs007/suvlim/writer/pb"
|
||||
"github.com/czs007/suvlim/writer/write_node"
|
||||
"sync"
|
||||
"testing"
|
||||
"writer/pb"
|
||||
"writer/write_node"
|
||||
)
|
||||
|
||||
func GetInsertMsg(collectionName string, partitionTag string, entityId int64) *pb.InsertOrDeleteMsg {
|
||||
|
|
@ -40,22 +40,14 @@ func TestInsert(t *testing.T) {
|
|||
wg := sync.WaitGroup{}
|
||||
wg.Add(3)
|
||||
//var wg sync.WaitGroup
|
||||
writerNode.InsertBatchData(ctx, insertMsgs, wg)
|
||||
data1 := writerNode.KvStore.GetData(ctx)
|
||||
//gtInsertBuffer := writerNode.GetInsertBuffer()
|
||||
println(len(data1))
|
||||
|
||||
writerNode.InsertBatchData(ctx, insertMsgs, &wg)
|
||||
var insertMsgs2 []*pb.InsertOrDeleteMsg
|
||||
for i := 120; i < 200; i++ {
|
||||
insertMsgs2 = append(insertMsgs2, GetInsertMsg("collection0", "tag02", int64(i)))
|
||||
}
|
||||
writerNode.InsertBatchData(ctx, insertMsgs2, wg)
|
||||
data2 := writerNode.KvStore.GetData(ctx)
|
||||
println(len(data2))
|
||||
writerNode.InsertBatchData(ctx, insertMsgs2, &wg)
|
||||
var deleteMsgs []*pb.InsertOrDeleteMsg
|
||||
deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 2))
|
||||
deleteMsgs = append(deleteMsgs, GetDeleteMsg("collection0", 120))
|
||||
writerNode.DeleteBatchData(ctx, deleteMsgs, wg)
|
||||
data3 := writerNode.KvStore.GetData(ctx)
|
||||
println(len(data3))
|
||||
writerNode.DeleteBatchData(ctx, deleteMsgs, &wg)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,21 +3,22 @@ package write_node
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
storage "github.com/czs007/suvlim/storage/pkg"
|
||||
"github.com/czs007/suvlim/storage/pkg/types"
|
||||
"github.com/czs007/suvlim/writer/message_client"
|
||||
"github.com/czs007/suvlim/writer/pb"
|
||||
"strconv"
|
||||
"sync"
|
||||
"writer/message_client"
|
||||
"writer/mock"
|
||||
"writer/pb"
|
||||
)
|
||||
|
||||
type SegmentIdInfo struct {
|
||||
CollectionName string
|
||||
EntityId int64
|
||||
SegmentIds *[]string
|
||||
SegmentIds [][]string
|
||||
}
|
||||
|
||||
type WriteNode struct {
|
||||
KvStore *mock.TikvStore
|
||||
KvStore *types.Store
|
||||
MessageClient *message_client.MessageClient
|
||||
TimeSync uint64
|
||||
}
|
||||
|
|
@ -26,20 +27,20 @@ func NewWriteNode(ctx context.Context,
|
|||
address string,
|
||||
topics []string,
|
||||
timeSync uint64) (*WriteNode, error) {
|
||||
kv, err := mock.NewTikvStore()
|
||||
mc := &message_client.MessageClient{}
|
||||
kv, err := storage.NewStore(context.Background(), types.MinIODriver)
|
||||
mc := message_client.MessageClient{}
|
||||
return &WriteNode{
|
||||
KvStore: kv,
|
||||
MessageClient: mc,
|
||||
KvStore: &kv,
|
||||
MessageClient: &mc,
|
||||
TimeSync: timeSync,
|
||||
}, err
|
||||
}
|
||||
|
||||
func (wn *WriteNode) InsertBatchData(ctx context.Context, data []*pb.InsertOrDeleteMsg, wg sync.WaitGroup) error {
|
||||
func (wn *WriteNode) InsertBatchData(ctx context.Context, data []*pb.InsertOrDeleteMsg, wg *sync.WaitGroup) error {
|
||||
var prefixKey string
|
||||
var suffixKey string
|
||||
var prefixKeys [][]byte
|
||||
var suffixKeys [][]byte
|
||||
var suffixKeys []string
|
||||
var binaryData [][]byte
|
||||
var timeStamp []uint64
|
||||
|
||||
|
|
@ -47,32 +48,36 @@ func (wn *WriteNode) InsertBatchData(ctx context.Context, data []*pb.InsertOrDel
|
|||
prefixKey = data[i].CollectionName + "-" + strconv.FormatUint(uint64(data[i].Uid), 10)
|
||||
suffixKey = strconv.FormatUint(uint64(data[i].SegmentId), 10)
|
||||
prefixKeys = append(prefixKeys, []byte(prefixKey))
|
||||
suffixKeys = append(suffixKeys, []byte(suffixKey))
|
||||
suffixKeys = append(suffixKeys, suffixKey)
|
||||
binaryData = append(binaryData, []byte(data[i].String()))
|
||||
timeStamp = append(timeStamp, uint64(data[i].Timestamp))
|
||||
}
|
||||
|
||||
error := (*wn.KvStore).PutRows(ctx, prefixKeys, timeStamp, suffixKeys, binaryData)
|
||||
error := (*wn.KvStore).PutRows(ctx, prefixKeys, binaryData, suffixKeys, timeStamp)
|
||||
if error != nil {
|
||||
fmt.Println("Can't insert data!")
|
||||
wg.Done()
|
||||
return error
|
||||
}
|
||||
wg.Done()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (wn *WriteNode) DeleteBatchData(ctx context.Context, data []*pb.InsertOrDeleteMsg, wg sync.WaitGroup) error {
|
||||
func (wn *WriteNode) DeleteBatchData(ctx context.Context, data []*pb.InsertOrDeleteMsg, wg *sync.WaitGroup) error {
|
||||
var segmentInfos []*SegmentIdInfo
|
||||
var prefixKey string
|
||||
var prefixKeys [][]byte
|
||||
var timeStamps []uint64
|
||||
var segmentIds [][]string
|
||||
|
||||
for i := 0; i < len(data); i++ {
|
||||
prefixKey = data[i].CollectionName + "-" + strconv.FormatUint(uint64(data[i].Uid), 10)
|
||||
prefixKeys = append(prefixKeys, []byte(prefixKey))
|
||||
timeStamps = append(timeStamps, uint64(data[i].Timestamp))
|
||||
segmentId, _ := (*wn.KvStore).GetSegments(ctx, []byte(prefixKey), uint64(data[i].Timestamp))
|
||||
segmentIds = append(segmentIds, segmentId)
|
||||
}
|
||||
segmentIds := (*wn.KvStore).GetSegment(ctx, prefixKeys)
|
||||
|
||||
for i := 0; i < len(prefixKeys); i++ {
|
||||
segmentInfos = append(segmentInfos, &SegmentIdInfo{
|
||||
CollectionName: data[i].CollectionName,
|
||||
|
|
@ -83,6 +88,8 @@ func (wn *WriteNode) DeleteBatchData(ctx context.Context, data []*pb.InsertOrDel
|
|||
err := (*wn.KvStore).DeleteRows(ctx, prefixKeys, timeStamps)
|
||||
if err != nil {
|
||||
fmt.Println("Can't delete data")
|
||||
wg.Done()
|
||||
return err
|
||||
}
|
||||
wg.Done()
|
||||
return nil
|
||||
|
|
@ -92,7 +99,7 @@ func (wn *WriteNode) UpdateTimeSync(timeSync uint64) {
|
|||
wn.TimeSync = timeSync
|
||||
}
|
||||
|
||||
func (wn *WriteNode) DoWriteNode(ctx context.Context, timeSync uint64, wg sync.WaitGroup) {
|
||||
func (wn *WriteNode) DoWriteNode(ctx context.Context, timeSync uint64, wg *sync.WaitGroup) {
|
||||
wg.Add(2)
|
||||
go wn.InsertBatchData(ctx, wn.MessageClient.InsertMsg, wg)
|
||||
go wn.DeleteBatchData(ctx, wn.MessageClient.DeleteMsg, wg)
|
||||
|
|
|
|||
Loading…
Reference in New Issue