milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

OTEL integration in cpp (#22175) 

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>
pull/22900/head
Enwei Jiao 2023-03-21 16:31:57 +08:00 committed by GitHub
parent 9efebb3df7
commit fd974df09f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 467 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
go.mod
View File

@ -49,7 +49,7 @@ require (
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
google.golang.org/grpc v1.51.0
google.golang.org/grpc v1.52.3
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f
google.golang.org/protobuf v1.28.1
gopkg.in/natefinch/lumberjack.v2 v2.0.0
@ -59,7 +59,6 @@ require (
)
require (
cloud.google.com/go v0.81.0 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
github.com/AthenZ/athenz v1.10.39 // indirect
github.com/DataDog/zstd v1.5.0 // indirect
@ -151,10 +150,10 @@ require (
go.etcd.io/etcd/client/v2 v2.305.5 // indirect
go.etcd.io/etcd/pkg/v3 v3.5.5 // indirect
go.etcd.io/etcd/raft/v3 v3.5.5 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0
go.opentelemetry.io/otel v1.11.2
go.opentelemetry.io/otel/sdk v1.11.2
go.opentelemetry.io/otel/trace v1.11.2
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0
go.opentelemetry.io/otel v1.13.0
go.opentelemetry.io/otel/sdk v1.13.0
go.opentelemetry.io/otel/trace v1.13.0
go.opentelemetry.io/proto/otlp v0.19.0 // indirect
go.uber.org/multierr v1.6.0
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
@ -165,10 +164,10 @@ require (
golang.org/x/text v0.7.0
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
gonum.org/v1/gonum v0.9.3 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 // indirect
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
@ -181,13 +180,16 @@ require (
github.com/aliyun/credentials-go v1.2.6
github.com/cockroachdb/errors v1.9.1
github.com/uber/jaeger-client-go v2.30.0+incompatible
go.opentelemetry.io/otel/exporters/jaeger v1.11.2
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2
go.opentelemetry.io/otel/exporters/jaeger v1.13.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.13.0
)
require (
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
github.com/alibabacloud-go/tea v1.1.8 // indirect
cloud.google.com/go/compute v1.12.1 // indirect
cloud.google.com/go/compute/metadata v0.2.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.0 // indirect
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
@ -202,10 +204,9 @@ require (
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/rogpeppe/go-internal v1.8.1 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 // indirect
go.opentelemetry.io/otel/metric v0.34.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 // indirect
go.opentelemetry.io/otel/metric v0.35.0 // indirect
)
replace (

55
go.sum
View File

@ -17,7 +17,6 @@ cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKP
cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk=
cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg=
cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8=
cloud.google.com/go v0.81.0 h1:at8Tk2zUz63cLPR0JPWm5vp77pEZmzxEQBEfRKn1VV8=
cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE=
@ -25,6 +24,10 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk=
cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
@ -846,36 +849,36 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0 h1:+uFejS4DCfNH6d3xODVIGsdhzgzhh45p9gpbHQMbdZI=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.37.0/go.mod h1:HSmzQvagH8pS2/xrK7ScWsk0vAMtRTGbMFgInXCi8Tc=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0 h1:g/BAN5o90Pr6D8xMRezjzGOHBpc15U+4oE53nZLiae4=
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.38.0/go.mod h1:+F41JBSkye7aYJELRvIMF0Z66reIwIOL0St75ZVwSJs=
go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU=
go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0=
go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI=
go.opentelemetry.io/otel/exporters/jaeger v1.11.2 h1:ES8/j2+aB+3/BUw51ioxa50V9btN1eew/2J7N7n1tsE=
go.opentelemetry.io/otel/exporters/jaeger v1.11.2/go.mod h1:nwcF/DK4Hk0auZ/a5vw20uMsaJSXbzeeimhN5f9d0Lc=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 h1:htgM8vZIF8oPSCxa341e3IZ4yr/sKxgu8KZYllByiVY=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw=
go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y=
go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg=
go.opentelemetry.io/otel/exporters/jaeger v1.13.0 h1:VAMoGujbVV8Q0JNM/cEbhzUIWWBxnEqH45HP9iBKN04=
go.opentelemetry.io/otel/exporters/jaeger v1.13.0/go.mod h1:fHwbmle6mBFJA1p2ZIhilvffCdq/dM5UTIiCOmEjS+w=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0 h1:pa05sNT/P8OsIQ8mPZKTIyiBuzS/xDGLVx+DCt0y6Vs=
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.13.0/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 h1:fqR1kli93643au1RKo0Uma3d2aPQKT+WBKfTSBaKbOc=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0 h1:Any/nVxaoMq1T2w0W85d6w5COlLuCCgOYKQhJJWEMwQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.13.0/go.mod h1:46vAP6RWfNn7EKov73l5KBFlNxz8kYlxR1woU+bJ4ZY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2 h1:ERwKPn9Aer7Gxsc0+ZlutlH1bEEAUXAUhqm3Y45ABbk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.11.2/go.mod h1:jWZUM2MWhWCJ9J9xVbRx7tzK1mXKpAlze4CeulycwVY=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2 h1:BhEVgvuE1NWLLuMLvC6sif791F45KFHi5GhOs1KunZU=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2/go.mod h1:bx//lU66dPzNT+Y0hHA12ciKoMOH9iixEwCqC1OeQWQ=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0 h1:Wz7UQn7/eIqZVDJbuNEM6PmqeA71cWXrWcXekP5HZgU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.13.0/go.mod h1:OhH1xvgA5jZW2M/S4PcvtDlFE1VULRRBsibBrKuJQGI=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.13.0 h1:rs3xmoGZsuHJxUUzX2dwYNDc7S0L68oEo2L/MvG5cyc=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.13.0/go.mod h1:gr0y6t58jZxp9WtIAGKXxXenDWC91hmZivlGoOag3+4=
go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8=
go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8=
go.opentelemetry.io/otel/metric v0.35.0 h1:aPT5jk/w7F9zW51L7WgRqNKDElBdyRLGuBtI5MX34e8=
go.opentelemetry.io/otel/metric v0.35.0/go.mod h1:qAcbhaTRFU6uG8QM7dDo7XvFsWcugziq/5YI065TokQ=
go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI=
go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU=
go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU=
go.opentelemetry.io/otel/sdk v1.13.0 h1:BHib5g8MvdqS65yo2vV1s6Le42Hm6rrw08qU6yz5JaM=
go.opentelemetry.io/otel/sdk v1.13.0/go.mod h1:YLKPx5+6Vx/o1TCUYYs+bpymtkmazOMT6zoRrC7AQ7I=
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk=
go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0=
go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA=
go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY=
go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw=
@ -1225,8 +1228,9 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0=
gonum.org/v1/gonum v0.9.3 h1:DnoIG+QAMaF5NvxnGe/oKsgKcAc6PcUyl8q0VetfQ8s=
@ -1312,8 +1316,9 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH
google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24=
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29 h1:DJUvgAPiJWeMBiT+RzBVcJGQN7bAEWS5UEoMshES9xs=
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c=
google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
@ -1341,8 +1346,8 @@ google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzI
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U=
google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww=
google.golang.org/grpc v1.52.3 h1:pf7sOysg4LdgBqduXveGKrcEwbStiK2rtfghdzlUYDQ=
google.golang.org/grpc v1.52.3/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY=
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec=
google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f/go.mod h1:gxndsbNG1n4TZcHGgsYEfVGnTxqfEdfiDv6/DADXX9o=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=

13
internal/core/conanfile.py
View File

@ -17,7 +17,7 @@ class MilvusConan(ConanFile):
"aws-sdk-cpp/1.9.234",
"benchmark/1.7.0",
"gtest/1.8.1",
"protobuf/3.21.9",
"protobuf/3.21.4",
"rapidxml/1.13",
"yaml-cpp/0.7.0",
"marisa/0.2.6",
@ -35,8 +35,9 @@ class MilvusConan(ConanFile):
"flex/2.6.4",
"xsimd/9.0.1",
"xz_utils/5.4.0",
"folly/2022.10.31.01@milvus/dev",
"velox/2023.02.07.01@milvus/dev",
"folly/2023.02.24@milvus/dev",
"velox/2023.02.24@milvus/dev",
"opentelemetry-cpp/1.8.1.1@milvus/dev",
)
generators = ("cmake", "cmake_find_package")
default_options = {
@ -49,12 +50,12 @@ class MilvusConan(ConanFile):
"aws-sdk-cpp:text-to-speech": False,
"aws-sdk-cpp:transfer": False,
"gtest:build_gmock": False,
"folly:use_sse4_2": True,
"boost:without_locale": False,
}
def configure(self):
# Macos M1 cannot use jemalloc
if self.settings.os == "Macos":
# Macos M1 cannot use jemalloc
if self.settings.arch not in ("x86_64", "x86"):
del self.options["folly"].use_sse4_2
@ -66,7 +67,6 @@ class MilvusConan(ConanFile):
self.options["boost"].without_graph = True
self.options["boost"].without_graph_parallel = True
self.options["boost"].without_nowide = True
self.options["boost"].without_locale = True
self.options["boost"].without_url = True
def imports(self):
@ -74,3 +74,4 @@ class MilvusConan(ConanFile):
self.copy("*.dll", "../lib", "lib")
self.copy("*.so*", "../lib", "lib")
self.copy("*", "../bin", "bin")
self.copy("*.proto", "../include", "include")

1
internal/core/src/common/CMakeLists.txt
View File

@ -19,6 +19,7 @@ set(COMMON_SRC
init_c.cpp
Common.cpp
RangeSearchHelper.cpp
Tracer.cpp
)
add_library(milvus_common SHARED ${COMMON_SRC})

87
internal/core/src/common/Tracer.cpp Normal file
View File

@ -0,0 +1,87 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "log/Log.h"
#include "Tracer.h"
#include "opentelemetry/exporters/ostream/span_exporter_factory.h"
#include "opentelemetry/exporters/jaeger/jaeger_exporter_factory.h"
#include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h"
#include "opentelemetry/sdk/trace/samplers/always_on.h"
#include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
#include "opentelemetry/sdk/trace/tracer_provider_factory.h"
#include "opentelemetry/sdk/trace/sampler.h"
#include "opentelemetry/sdk/trace/samplers/parent.h"
#include "opentelemetry/sdk/resource/resource.h"
#include "opentelemetry/trace/span_context.h"
#include "opentelemetry/trace/span_metadata.h"
namespace milvus::tracer {
namespace trace = opentelemetry::trace;
namespace nostd = opentelemetry::nostd;
namespace trace_sdk = opentelemetry::sdk::trace;
namespace resource = opentelemetry::sdk::resource;
namespace jaeger = opentelemetry::exporter::jaeger;
namespace ostream = opentelemetry::exporter::trace;
namespace otlp = opentelemetry::exporter::otlp;
void
initTelementry(TraceConfig* config) {
std::unique_ptr<opentelemetry::sdk::trace::SpanExporter> exporter;
if (config->exporter == "stdout") {
exporter = ostream::OStreamSpanExporterFactory::Create();
} else if (config->exporter == "jaeger") {
auto opts = jaeger::JaegerExporterOptions{};
opts.transport_format = jaeger::TransportFormat::kThriftHttp;
opts.endpoint = config->jaegerURL;
exporter = jaeger::JaegerExporterFactory::Create(opts);
} else if (config->exporter == "otlp") {
auto opts = otlp::OtlpGrpcExporterOptions{};
opts.endpoint = config->otlpEndpoint;
exporter = otlp::OtlpGrpcExporterFactory::Create(opts);
} else {
LOG_SEGCORE_INFO_ << "Empty Trace";
}
auto processor =
trace_sdk::BatchSpanProcessorFactory::Create(std::move(exporter), {});
resource::ResourceAttributes attributes = {{"service.name", "segcore"},
{"NodeID", config->nodeID}};
auto resource = resource::Resource::Create(attributes);
auto sampler = std::make_unique<trace_sdk::ParentBasedSampler>(
std::make_shared<trace_sdk::AlwaysOnSampler>());
std::shared_ptr<trace::TracerProvider> provider =
trace_sdk::TracerProviderFactory::Create(
std::move(processor), resource, std::move(sampler));
trace::Provider::SetTracerProvider(provider);
}
std::shared_ptr<trace::Tracer>
GetTracer() {
auto provider = trace::Provider::GetTracerProvider();
return provider->GetTracer("segcore", OPENTELEMETRY_SDK_VERSION);
}
std::shared_ptr<trace::Span>
StartSpan(std::string name, TraceContext* parentCtx) {
trace::StartSpanOptions opts;
if (parentCtx != nullptr && parentCtx->traceID != nullptr &&
parentCtx->spanID != nullptr) {
opts.parent = trace::SpanContext(
trace::TraceId({parentCtx->traceID, trace::TraceId::kSize}),
trace::SpanId({parentCtx->spanID, trace::SpanId::kSize}),
trace::TraceFlags(parentCtx->flag),
true);
}
return GetTracer()->StartSpan(name, opts);
}
} // namespace milvus::tracer

41
internal/core/src/common/Tracer.h Normal file
View File

@ -0,0 +1,41 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "opentelemetry/sdk/version/version.h"
#include "opentelemetry/trace/provider.h"
namespace milvus::tracer {
struct TraceConfig {
std::string exporter;
int sampleFraction;
std::string jaegerURL;
std::string otlpEndpoint;
int nodeID;
};
struct TraceContext {
const uint8_t* traceID;
const uint8_t* spanID;
uint8_t flag;
};
namespace trace = opentelemetry::trace;
void
initTelementry(TraceConfig* config);
std::shared_ptr<trace::Tracer>
GetTracer();
std::shared_ptr<trace::Span>
StartSpan(std::string name, TraceContext* ctx = nullptr);
} // namespace milvus::tracer

19
internal/core/src/common/init_c.cpp
View File

@ -14,6 +14,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <mutex>
#include "common/init_c.h"
@ -22,8 +23,11 @@
#include "config/ConfigChunkManager.h"
#include "common/Slice.h"
#include "common/Common.h"
#include "common/Tracer.h"
#include "log/Log.h"
std::once_flag flag1, flag2, flag3, flag4;
std::once_flag traceFlag;
void
InitLocalRootPath(const char* root_path) {
@ -55,3 +59,18 @@ InitCpuNum(const int value) {
std::call_once(
flag4, [](int value) { milvus::SetCpuNum(value); }, value);
}
void
InitTrace(CTraceConfig* config) {
auto traceConfig = milvus::tracer::TraceConfig{config->exporter,
config->sampleFraction,
config->jaegerURL,
config->otlpEndpoint,
config->nodeID};
std::call_once(
traceFlag,
[](milvus::tracer::TraceConfig* c) {
milvus::tracer::initTelementry(c);
},
&traceConfig);
}

4
internal/core/src/common/init_c.h
View File

@ -22,6 +22,7 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "type_c.h"
void
InitIndexSliceSize(const int64_t);
@ -35,6 +36,9 @@ InitCpuNum(const int);
void
InitLocalRootPath(const char*);
void
InitTrace(CTraceConfig* config);
#ifdef __cplusplus
};
#endif

14
internal/core/src/common/type_c.h
View File

@ -98,6 +98,20 @@ typedef struct CStorageConfig {
bool useIAM;
} CStorageConfig;
typedef struct CTraceConfig {
const char* exporter;
int sampleFraction;
const char* jaegerURL;
const char* otlpEndpoint;
int nodeID;
} CTraceConfig;
typedef struct CTraceContext {
const uint8_t* traceID;
const uint8_t* spanID;
uint8_t flag;
} CTraceContext;
#ifdef __cplusplus
}
#endif

2
internal/core/src/pb/common.pb.h
View File

@ -13,7 +13,7 @@
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3021009 < PROTOBUF_MIN_PROTOC_VERSION
#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.

2
internal/core/src/pb/index_cgo_msg.pb.h
View File

@ -13,7 +13,7 @@
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3021009 < PROTOBUF_MIN_PROTOC_VERSION
#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.

2
internal/core/src/pb/plan.pb.h
View File

@ -13,7 +13,7 @@
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3021009 < PROTOBUF_MIN_PROTOC_VERSION
#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.

2
internal/core/src/pb/schema.pb.h
View File

@ -13,7 +13,7 @@
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3021009 < PROTOBUF_MIN_PROTOC_VERSION
#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.

2
internal/core/src/pb/segcore.pb.h
View File

@ -13,7 +13,7 @@
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3021009 < PROTOBUF_MIN_PROTOC_VERSION
#if 3021004 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.

42
internal/core/src/segcore/segment_c.cpp
View File

@ -14,6 +14,7 @@
#include "common/CGoHelper.h"
#include "common/LoadInfo.h"
#include "common/Types.h"
#include "common/Tracer.h"
#include "common/type_c.h"
#include "google/protobuf/text_format.h"
#include "index/IndexInfo.h"
@ -25,7 +26,7 @@
////////////////////////////// common interfaces //////////////////////////////
CSegmentInterface
NewSegment(CCollection collection, SegmentType seg_type, int64_t segment_id) {
auto col = (milvus::segcore::Collection*)collection;
auto col = static_cast<milvus::segcore::Collection*>(collection);
std::unique_ptr<milvus::segcore::SegmentInterface> segment;
switch (seg_type) {
@ -42,23 +43,23 @@ NewSegment(CCollection collection, SegmentType seg_type, int64_t segment_id) {
segment_id);
break;
default:
LOG_SEGCORE_ERROR_ << "invalid segment type " << (int32_t)seg_type;
LOG_SEGCORE_ERROR_ << "invalid segment type "
<< static_cast<int32_t>(seg_type);
break;
}
return (void*)segment.release();
return segment.release();
}
void
DeleteSegment(CSegmentInterface c_segment) {
// TODO: use dynamic cast, and return c status
auto s = (milvus::segcore::SegmentInterface*)c_segment;
auto s = static_cast<milvus::segcore::SegmentInterface*>(c_segment);
delete s;
}
void
DeleteSearchResult(CSearchResult search_result) {
auto res = (milvus::SearchResult*)search_result;
auto res = static_cast<milvus::SearchResult*>(search_result);
delete res;
}
@ -66,6 +67,7 @@ CStatus
Search(CSegmentInterface c_segment,
CSearchPlan c_plan,
CPlaceholderGroup c_placeholder_group,
CTraceContext c_trace,
uint64_t timestamp,
CSearchResult* result) {
try {
@ -73,6 +75,11 @@ Search(CSegmentInterface c_segment,
auto plan = (milvus::query::Plan*)c_plan;
auto phg_ptr = reinterpret_cast<const milvus::query::PlaceholderGroup*>(
c_placeholder_group);
auto ctx = milvus::tracer::TraceContext{
c_trace.traceID, c_trace.spanID, c_trace.flag};
auto span = milvus::tracer::StartSpan("SegcoreSearch", &ctx);
auto search_result = segment->Search(plan, phg_ptr, timestamp);
if (!milvus::PositivelyRelated(
plan->plan_node_->search_info_.metric_type_)) {
@ -81,6 +88,8 @@ Search(CSegmentInterface c_segment,
}
}
*result = search_result.release();
span->End();
return milvus::SuccessCStatus();
} catch (std::exception& e) {
return milvus::FailureCStatus(UnexpectedError, e.what());
@ -89,7 +98,7 @@ Search(CSegmentInterface c_segment,
void
DeleteRetrieveResult(CRetrieveResult* retrieve_result) {
std::free((void*)(retrieve_result->proto_blob));
std::free(const_cast<void*>(retrieve_result->proto_blob));
}
CStatus
@ -98,8 +107,9 @@ Retrieve(CSegmentInterface c_segment,
uint64_t timestamp,
CRetrieveResult* result) {
try {
auto segment = (const milvus::segcore::SegmentInterface*)c_segment;
auto plan = (const milvus::query::RetrievePlan*)c_plan;
auto segment =
static_cast<const milvus::segcore::SegmentInterface*>(c_segment);
auto plan = static_cast<const milvus::query::RetrievePlan*>(c_plan);
auto retrieve_result = segment->Retrieve(plan, timestamp);
auto size = retrieve_result->ByteSizeLong();
@ -116,14 +126,14 @@ Retrieve(CSegmentInterface c_segment,
int64_t
GetMemoryUsageInBytes(CSegmentInterface c_segment) {
auto segment = (milvus::segcore::SegmentInterface*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto mem_size = segment->GetMemoryUsageInBytes();
return mem_size;
}
int64_t
GetRowCount(CSegmentInterface c_segment) {
auto segment = (milvus::segcore::SegmentInterface*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto row_count = segment->get_row_count();
return row_count;
}
@ -156,7 +166,7 @@ Insert(CSegmentInterface c_segment,
const uint8_t* data_info,
const uint64_t data_info_len) {
try {
auto segment = (milvus::segcore::SegmentGrowing*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentGrowing*>(c_segment);
auto insert_data = std::make_unique<milvus::InsertData>();
auto suc = insert_data->ParseFromArray(data_info, data_info_len);
AssertInfo(suc, "failed to parse insert data from records");
@ -172,7 +182,7 @@ Insert(CSegmentInterface c_segment,
CStatus
PreInsert(CSegmentInterface c_segment, int64_t size, int64_t* offset) {
try {
auto segment = (milvus::segcore::SegmentGrowing*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentGrowing*>(c_segment);
*offset = segment->PreInsert(size);
return milvus::SuccessCStatus();
} catch (std::exception& e) {
@ -187,7 +197,7 @@ Delete(CSegmentInterface c_segment,
const uint8_t* ids,
const uint64_t ids_size,
const uint64_t* timestamps) {
auto segment = (milvus::segcore::SegmentInterface*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto pks = std::make_unique<milvus::proto::schema::IDs>();
auto suc = pks->ParseFromArray(ids, ids_size);
AssertInfo(suc, "failed to parse pks from ids");
@ -202,7 +212,7 @@ Delete(CSegmentInterface c_segment,
int64_t
PreDelete(CSegmentInterface c_segment, int64_t size) {
auto segment = (milvus::segcore::SegmentInterface*)c_segment;
auto segment = static_cast<milvus::segcore::SegmentInterface*>(c_segment);
return segment->PreDelete(size);
}
@ -263,7 +273,7 @@ UpdateSealedSegmentIndex(CSegmentInterface c_segment,
dynamic_cast<milvus::segcore::SegmentSealed*>(segment_interface);
AssertInfo(segment != nullptr, "segment conversion failed");
auto load_index_info =
(milvus::segcore::LoadIndexInfo*)c_load_index_info;
static_cast<milvus::segcore::LoadIndexInfo*>(c_load_index_info);
segment->LoadIndex(*load_index_info);
return milvus::SuccessCStatus();
} catch (std::exception& e) {

1
internal/core/src/segcore/segment_c.h
View File

@ -41,6 +41,7 @@ CStatus
Search(CSegmentInterface c_segment,
CSearchPlan c_plan,
CPlaceholderGroup c_placeholder_group,
CTraceContext c_trace,
uint64_t timestamp,
CSearchResult* result);

1
internal/core/unittest/CMakeLists.txt
View File

@ -47,6 +47,7 @@ set(MILVUS_TEST_FILES
test_utils.cpp
test_data_codec.cpp
test_range_search_sort.cpp
test_tracer.cpp
)
if ( BUILD_DISK_ANN STREQUAL "ON" )

141
internal/core/unittest/test_c_api.cpp
View File

@ -1004,13 +1004,13 @@ TEST(CApiTest, SearchTest) {
placeholderGroups.push_back(placeholderGroup);
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, N + ts_offset, &search_result);
auto res = Search(
segment, plan, placeholderGroup, {}, N + ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
CSearchResult search_result2;
auto res2 =
Search(segment, plan, placeholderGroup, ts_offset, &search_result2);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result2);
ASSERT_EQ(res2.error_code, Success);
DeleteSearchPlan(plan);
@ -1075,6 +1075,7 @@ TEST(CApiTest, SearchTestWithExpr) {
auto res = Search(segment,
plan,
placeholderGroup,
{},
dataset.timestamps_[0],
&search_result);
ASSERT_EQ(res.error_code, Success);
@ -1154,11 +1155,6 @@ TEST(CApiTest, GetMemoryUsageInBytesTest) {
insert_data.size());
ASSERT_EQ(res.error_code, Success);
auto memory_usage_size = GetMemoryUsageInBytes(segment);
// std::cout << "new_memory_usage_size = " << memory_usage_size << std::endl;
// TODO:: assert
// ASSERT_EQ(memory_usage_size, 2785280);
DeleteCollection(collection);
DeleteSegment(segment);
}
@ -1363,7 +1359,7 @@ TEST(CApiTest, ReudceNullResult) {
std::vector<CSearchResult> results;
CSearchResult res;
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res);
ASSERT_EQ(status.error_code, Success);
results.push_back(res);
CSearchResultDataBlobs cSearchResultData;
@ -1453,10 +1449,10 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
std::vector<CSearchResult> results;
CSearchResult res1, res2;
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res1);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res2);
ASSERT_EQ(status.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -1486,13 +1482,13 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
std::vector<CSearchResult> results;
CSearchResult res1, res2, res3;
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res1);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res2);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res3);
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res3);
ASSERT_EQ(status.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -1580,10 +1576,10 @@ testReduceSearchWithExpr(int N, int topK, int num_queries) {
CSearchResult res1;
CSearchResult res2;
auto res = Search(
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res1);
segment, plan, placeholderGroup, {}, dataset.timestamps_[N - 1], &res1);
ASSERT_EQ(res.error_code, Success);
res = Search(
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res2);
segment, plan, placeholderGroup, {}, dataset.timestamps_[N - 1], &res2);
ASSERT_EQ(res.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -1821,8 +1817,12 @@ TEST(CApiTest, Indexing_Without_Predicate) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -1882,6 +1882,7 @@ TEST(CApiTest, Indexing_Without_Predicate) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -1966,8 +1967,12 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2027,6 +2032,7 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2127,8 +2133,12 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2188,6 +2198,7 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2301,8 +2312,12 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2362,6 +2377,7 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2458,8 +2474,12 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2519,6 +2539,7 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2625,8 +2646,12 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2686,6 +2711,7 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2784,8 +2810,12 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -2846,6 +2876,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2958,8 +2989,12 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_TRUE(res_before_load_index.error_code == Success)
<< res_before_load_index.error_msg;
@ -3020,6 +3055,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3118,8 +3154,12 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -3179,6 +3219,7 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3308,8 +3349,12 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
Timestamp time = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_smallIndex);
auto res_before_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
// load index to segment
@ -3369,6 +3414,7 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3594,6 +3640,7 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
auto res_after_load_index = Search(sealed_segment.get(),
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3715,14 +3762,14 @@ TEST(CApiTest, SealedSegment_search_without_predicates) {
std::vector<CPlaceholderGroup> placeholderGroups;
placeholderGroups.push_back(placeholderGroup);
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, N + ts_offset, &search_result);
auto res = Search(
segment, plan, placeholderGroup, {}, N + ts_offset, &search_result);
std::cout << res.error_msg << std::endl;
ASSERT_EQ(res.error_code, Success);
CSearchResult search_result2;
auto res2 =
Search(segment, plan, placeholderGroup, ts_offset, &search_result2);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result2);
ASSERT_EQ(res2.error_code, Success);
DeleteSearchPlan(plan);
@ -3903,8 +3950,12 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
}
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(
segment, plan, placeholderGroup, time, &c_search_result_on_bigIndex);
auto res_after_load_index = Search(segment,
plan,
placeholderGroup,
{},
time,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
auto search_result_on_bigIndex = (SearchResult*)c_search_result_on_bigIndex;
@ -4144,7 +4195,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_WHEN_IP) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, ts_offset, &search_result);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4212,7 +4263,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_IP) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, ts_offset, &search_result);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4279,7 +4330,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_WHEN_L2) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, ts_offset, &search_result);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4347,7 +4398,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_L2) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, ts_offset, &search_result);
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);

73
internal/core/unittest/test_tracer.cpp Normal file
View File

@ -0,0 +1,73 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include <memory>
#include <random>
#include "common/Tracer.h"
#include "exceptions/EasyAssert.h"
using namespace milvus;
using namespace milvus::tracer;
using namespace opentelemetry::trace;
TEST(Tracer, Init) {
auto config = std::make_shared<TraceConfig>();
config->exporter = "stdout";
config->nodeID = 1;
initTelementry(config.get());
auto span = StartSpan("test");
Assert(span->IsRecording());
config = std::make_shared<TraceConfig>();
config->exporter = "jaeger";
config->jaegerURL = "http://localhost:14268/api/traces";
config->nodeID = 1;
initTelementry(config.get());
span = StartSpan("test");
Assert(span->IsRecording());
}
TEST(Tracer, Span) {
auto config = std::make_shared<TraceConfig>();
config->exporter = "stdout";
config->nodeID = 1;
initTelementry(config.get());
auto ctx = std::make_shared<TraceContext>();
ctx->traceID = new uint8_t[16]{0x01,
0x23,
0x45,
0x67,
0x89,
0xab,
0xcd,
0xef,
0xfe,
0xdc,
0xba,
0x98,
0x76,
0x54,
0x32,
0x10};
ctx->spanID =
new uint8_t[8]{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef};
ctx->flag = 1;
auto span = StartSpan("test", ctx.get());
Assert(span->GetContext().trace_id() == trace::TraceId({ctx->traceID, 16}));
delete[] ctx->traceID;
delete[] ctx->spanID;
}

2
internal/querynode/query_node.go
View File

@ -243,6 +243,8 @@ func (node *QueryNode) InitSegcore() {
if len(mmapDirPath) > 0 {
log.Info("mmap enabled", zap.String("dir", mmapDirPath))
}
initcore.InitTraceConfig(Params)
}
// Init function init historical and streaming module to manage segments

2
internal/querynode/retrieve.go
View File

@ -38,7 +38,7 @@ func retrieveOnSegments(ctx context.Context, replica ReplicaInterface, segType s
}
return nil, err
}
result, err := seg.retrieve(plan)
result, err := seg.retrieve(ctx, plan)
if err != nil {
return nil, err
}

28
internal/querynode/segment.go
View File

@ -37,6 +37,7 @@ import (
bloom "github.com/bits-and-blooms/bloom/v3"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"go.opentelemetry.io/otel/trace"
"go.uber.org/atomic"
"go.uber.org/zap"
@ -299,15 +300,6 @@ func (s *Segment) getMemSize() int64 {
}
func (s *Segment) search(ctx context.Context, searchReq *searchRequest) (*SearchResult, error) {
/*
CStatus
Search(void* plan,
void* placeholder_groups,
uint64_t* timestamps,
int num_groups,
long int* result_ids,
float* result_distances);
*/
s.mut.RLock()
defer s.mut.RUnlock()
if !s.healthy() {
@ -325,9 +317,21 @@ func (s *Segment) search(ctx context.Context, searchReq *searchRequest) (*Search
zap.Int64("segmentID", s.segmentID),
zap.String("segmentType", s.segmentType.String()),
zap.Bool("loadIndex", loadIndex))
span := trace.SpanFromContext(ctx)
traceID := span.SpanContext().TraceID()
spanID := span.SpanContext().SpanID()
traceCtx := C.CTraceContext{
traceID: (*C.uint8_t)(unsafe.Pointer(&traceID[0])),
spanID: (*C.uint8_t)(unsafe.Pointer(&spanID[0])),
flag: C.uchar(span.SpanContext().TraceFlags()),
}
tr := timerecord.NewTimeRecorder("cgoSearch")
status := C.Search(s.segmentPtr, searchReq.plan.cSearchPlan, searchReq.cPlaceholderGroup,
status := C.Search(s.segmentPtr,
searchReq.plan.cSearchPlan,
searchReq.cPlaceholderGroup,
traceCtx,
C.uint64_t(searchReq.timestamp), &searchResult.cSearchResult)
metrics.QueryNodeSQSegmentLatencyInCore.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel).
Observe(float64(tr.CtxElapse(ctx, "finish cgoSearch").Milliseconds()))
@ -342,7 +346,7 @@ func (s *Segment) search(ctx context.Context, searchReq *searchRequest) (*Search
return &searchResult, nil
}
func (s *Segment) retrieve(plan *RetrievePlan) (*segcorepb.RetrieveResults, error) {
func (s *Segment) retrieve(ctx context.Context, plan *RetrievePlan) (*segcorepb.RetrieveResults, error) {
s.mut.RLock()
defer s.mut.RUnlock()
if !s.healthy() {
@ -355,7 +359,7 @@ func (s *Segment) retrieve(plan *RetrievePlan) (*segcorepb.RetrieveResults, erro
tr := timerecord.NewTimeRecorder("cgoRetrieve")
status := C.Retrieve(s.segmentPtr, plan.cRetrievePlan, ts, &retrieveResult.cRetrieveResult)
metrics.QueryNodeSQSegmentLatencyInCore.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()),
metrics.QueryLabel).Observe(float64(tr.ElapseSpan().Milliseconds()))
metrics.QueryLabel).Observe(float64(tr.CtxElapse(ctx, "finish cgoRetrieve").Milliseconds()))
log.Debug("do retrieve on segment",
zap.Int64("msgID", plan.msgID),
zap.Int64("segmentID", s.segmentID), zap.String("segmentType", s.segmentType.String()))

5
internal/querynode/segment_test.go
View File

@ -191,7 +191,8 @@ func TestSegment_retrieve(t *testing.T) {
defer plan.delete()
assert.NoError(t, err)
res, err := segment.retrieve(plan)
ctx := context.Background()
res, err := segment.retrieve(ctx, plan)
assert.NoError(t, err)
assert.Equal(t, res.GetFieldsData()[0].GetScalars().Data.(*schemapb.ScalarField_IntData).IntData.Data, []int32{1, 2, 3})
@ -204,7 +205,7 @@ func TestSegment_retrieve(t *testing.T) {
err = segment.segmentInsert(offset, insertMsg.RowIDs, insertMsg.Timestamps, insertRecord)
require.NoError(t, err)
res, err := segment.retrieve(plan)
res, err := segment.retrieve(ctx, plan)
assert.NoError(t, err)
assert.Equal(t, []int32{1, 1, 2, 2, 3, 3}, res.GetFieldsData()[0].GetScalars().Data.(*schemapb.ScalarField_IntData).IntData.Data)

5
internal/tracer/tracer.go
View File

@ -17,6 +17,8 @@
package tracer
import (
"context"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/internal/log"
@ -24,6 +26,7 @@ import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
@ -41,6 +44,8 @@ func Init() {
case "jaeger":
exp, err = jaeger.New(jaeger.WithCollectorEndpoint(
jaeger.WithEndpoint(params.TraceCfg.JaegerURL.GetValue())))
case "otlp":
exp, err = otlptracegrpc.New(context.Background(), otlptracegrpc.WithEndpoint(params.TraceCfg.OtlpEndpoint.GetValue()))
case "stdout":
exp, err = stdout.New()
default:

11
internal/util/initcore/init_storage_config.go → internal/util/initcore/init_core.go
View File

@ -39,3 +39,14 @@ func InitLocalStorageConfig(params *paramtable.ComponentParam) {
C.InitLocalRootPath(CLocalRootPath)
C.free(unsafe.Pointer(CLocalRootPath))
}
func InitTraceConfig(params *paramtable.ComponentParam) {
config := C.CTraceConfig{
exporter: C.CString(params.TraceCfg.Exporter.GetValue()),
sampleFraction: C.int(params.TraceCfg.SampleFraction.GetAsInt()),
jaegerURL: C.CString(params.TraceCfg.JaegerURL.GetValue()),
otlpEndpoint: C.CString(params.TraceCfg.OtlpEndpoint.GetValue()),
nodeID: C.int(paramtable.GetNodeID()),
}
C.InitTrace(&config)
}

7
internal/util/paramtable/component_param.go
View File

@ -615,6 +615,7 @@ type traceConfig struct {
Exporter ParamItem `refreshable:"false"`
SampleFraction ParamItem `refreshable:"false"`
JaegerURL ParamItem `refreshable:"false"`
OtlpEndpoint ParamItem `refreshable:"false"`
}
func (t *traceConfig) init(base *BaseTable) {
@ -645,6 +646,12 @@ Fractions >= 1 will always sample. Fractions < 0 are treated as zero.`,
Export: true,
}
t.JaegerURL.Init(base.mgr)
t.OtlpEndpoint = ParamItem{
Key: "trace.otlp.endpoint",
Version: "2.3.0",
}
t.OtlpEndpoint.Init(base.mgr)
}
type logConfig struct {

2
scripts/check_proto_product.sh
View File

@ -29,7 +29,7 @@ if [[ $(uname -s) == "Darwin" ]]; then
export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH"
fi
check_result=$(git status | grep -E "*pb.go|*pb.cc|*pb.h")
check_result=$(git status | grep -E ".*pb.go|.*pb.cc|.*pb.h")
echo "check_result: $check_result"
if test -z "$check_result"; then
exit 0

2
scripts/core_build.sh
View File

@ -199,7 +199,7 @@ case "${unameOut}" in
export CXX="${llvm_prefix}/bin/clang++"
export CFLAGS=-Wno-deprecated-declarations
export CXXFLAGS=-Wno-deprecated-declarations
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=15 -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; }
conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=15 -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; }
;;
Linux*)
echo "Running on ${OS_NAME}"