milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Geospatial Data Type and GIS Function Support for milvus server (#35990) 

issue:https://github.com/milvus-io/milvus/issues/27576

# Main Goals
1. Create and describe collections with geospatial fields, enabling both
client and server to recognize and process geo fields.
2. Insert geospatial data as payload values in the insert binlog, and
print the values for verification.
3. Load segments containing geospatial data into memory.
4. Ensure query outputs can display geospatial data.
5. Support filtering on GIS functions for geospatial columns.

# Solution
1. **Add Type**: Modify the Milvus core by adding a Geospatial type in
both the C++ and Go code layers, defining the Geospatial data structure
and the corresponding interfaces.
2. **Dependency Libraries**: Introduce necessary geospatial data
processing libraries. In the C++ source code, use Conan package
management to include the GDAL library. In the Go source code, add the
go-geom library to the go.mod file.
3. **Protocol Interface**: Revise the Milvus protocol to provide
mechanisms for Geospatial message serialization and deserialization.
4. **Data Pipeline**: Facilitate interaction between the client and
proxy using the WKT format for geospatial data. The proxy will convert
all data into WKB format for downstream processing, providing column
data interfaces, segment encapsulation, segment loading, payload
writing, and cache block management.
5. **Query Operators**: Implement simple display and support for filter
queries. Initially, focus on filtering based on spatial relationships
for a single column of geospatial literal values, providing parsing and
execution for query expressions.
6. **Client Modification**: Enable the client to handle user input for
geospatial data and facilitate end-to-end testing.Check the modification
in pymilvus.

---------

Signed-off-by: tasty-gumi <1021989072@qq.com>
pull/37338/head
Hao Tan 2024-10-31 20:58:20 +08:00 committed by GitHub
parent 4bac2eb13e
commit 67c4340565
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
75 changed files with 3404 additions and 606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
go.mod
View File

@ -6,10 +6,11 @@ toolchain go1.22.0
require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0
github.com/aliyun/credentials-go v1.2.7
github.com/apache/arrow/go/v12 v12.0.1
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect
github.com/bits-and-blooms/bloom/v3 v3.0.1
github.com/blang/semver/v4 v4.0.0
github.com/casbin/casbin/v2 v2.44.2
@ -36,6 +37,7 @@ require (
github.com/spf13/cast v1.3.1
github.com/spf13/viper v1.8.1
github.com/stretchr/testify v1.9.0
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.865
github.com/tikv/client-go/v2 v2.0.4
go.etcd.io/etcd/api/v3 v3.5.5
@ -61,7 +63,7 @@ require (
github.com/antlr4-go/antlr/v4 v4.13.1
github.com/bits-and-blooms/bitset v1.10.0
github.com/bytedance/sonic v1.12.2
github.com/cenkalti/backoff/v4 v4.2.1
github.com/cenkalti/backoff/v4 v4.3.0
github.com/cockroachdb/redact v1.1.3
github.com/goccy/go-json v0.10.3
github.com/greatroar/blobloom v0.0.0-00010101000000-000000000000
@ -71,6 +73,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/remeh/sizedwaitgroup v1.0.0
github.com/tidwall/gjson v1.17.1
github.com/twpayne/go-geom v1.4.0
github.com/valyala/fastjson v1.6.4
github.com/zeebo/xxh3 v1.0.2
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0
@ -96,7 +99,6 @@ require (
github.com/alibabacloud-go/debug v0.0.0-20190504072949-9472017b5c68 // indirect
github.com/alibabacloud-go/tea v1.1.8 // indirect
github.com/andybalholm/brotli v1.0.4 // indirect
github.com/apache/pulsar-client-go v0.6.1-0.20210728062540-29414db801a7 // indirect
github.com/apache/thrift v0.18.1 // indirect
github.com/ardielle/ardielle-go v1.5.2 // indirect
github.com/benesch/cgosymbolizer v0.0.0-20190515212042-bec6fe6e597b // indirect
@ -147,7 +149,7 @@ require (
github.com/gorilla/websocket v1.4.2 // indirect
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
@ -210,7 +212,6 @@ require (
github.com/streamnative/pulsarctl v0.5.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
@ -233,13 +234,13 @@ require (
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.13.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.20.0 // indirect
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0 // indirect
go.opentelemetry.io/otel/metric v1.28.0 // indirect
go.opentelemetry.io/otel/sdk v1.28.0 // indirect
go.opentelemetry.io/proto/otlp v1.0.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/automaxprocs v1.5.3 // indirect
golang.org/x/arch v0.3.0 // indirect
golang.org/x/mod v0.17.0 // indirect
@ -250,7 +251,7 @@ require (
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
gonum.org/v1/gonum v0.11.0 // indirect
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect

66
go.sum
View File

@ -61,14 +61,15 @@ github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY
github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1 h1:E+OJmp2tPvt1W+amx48v1eqbjDYsgN+RzP4q16yV5eM=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.11.1/go.mod h1:a6xsAQUZg+VsS3TJ05SRp524Hs4pZ/AeFSr5ENf0Yjo=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0 h1:U2rTu3Ef+7w9FHKIAXM6ZyqF3UOWJZ12zIm8zECAFfg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.6.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0 h1:jBQA3cKT4L2rWMpgE7Yt3Hwh2aUj8KXjIGLxjHeYNNo=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.8.0/go.mod h1:4OG6tQ9EOP/MT0NMjDlRzWoVFxfu9rN9B2X+tlSVktg=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0/go.mod h1:c+Lifp3EDEamAkPVzMooRNOK6CZjNSdEnf1A7jsI9u4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU=
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@ -77,6 +78,8 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbi
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
github.com/DATA-DOG/go-sqlmock v1.3.2 h1:2L2f5t3kKnCLxnClDD/PrDfExFFa1wjESgxHG/B1ibo=
github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU=
@ -84,6 +87,11 @@ github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw=
github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
github.com/SimFG/expr v0.0.0-20241029080923-21854792f532 h1:fxpBc+wl14Ky70AVgCJ4muWzAzXcxoerlRxA1F69S0M=
@ -150,9 +158,10 @@ github.com/casbin/casbin/v2 v2.44.2 h1:mlWtgbX872r707frOq+REaHzfvsl+qQw0Eq+ekzJ7
github.com/casbin/casbin/v2 v2.44.2/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg=
github.com/casbin/json-adapter/v2 v2.0.0 h1:nOCN3TK1CJKSNQQ/MnakbU9/cUcNR3N0AxBDnEBLSDI=
github.com/casbin/json-adapter/v2 v2.0.0/go.mod h1:LvsfPXXr8CD0ZFucAxawcY9Xb0FtLk3mozJ1qcSTUD4=
github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
@ -195,6 +204,7 @@ github.com/confluentinc/confluent-kafka-go v1.9.1 h1:L3aW6KvTyrq/+BOMnDm9xJylhAE
github.com/confluentinc/confluent-kafka-go v1.9.1/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo=
github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@ -224,6 +234,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@ -348,8 +359,6 @@ github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17w
github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4=
github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@ -437,6 +446,7 @@ github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8I
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -462,8 +472,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c=
github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU=
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0=
github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8=
@ -496,11 +506,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
@ -562,6 +574,7 @@ github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -584,6 +597,8 @@ github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awS
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
@ -627,8 +642,6 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34 h1:Fwxpg98128gfWRbQ1A3PMP9o2IfYZk7RSEy8rcoCWDA=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7 h1:HwAitQk+V59QdYUwwVVYHTujd4QZrebg2Cc2hmcjhAg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
@ -644,6 +657,7 @@ github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEp
github.com/minio/minio-go/v7 v7.0.73 h1:qr2vi96Qm7kZ4v7LLebjte+MQh621fFWnv93p12htEo=
github.com/minio/minio-go/v7 v7.0.73/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@ -653,6 +667,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -692,11 +707,15 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ=
github.com/panjf2000/ants/v2 v2.7.2 h1:2NUt9BaZFO5kQzrieOmK/wdb/tQ/K+QHaxN8sOgD63U=
github.com/panjf2000/ants/v2 v2.7.2/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@ -809,6 +828,7 @@ github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@ -895,6 +915,11 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twpayne/go-geom v1.4.0 h1:gtI+ClrojsGDl0SXXNAwPUtPxanyodQzXg8mliV3d/w=
github.com/twpayne/go-geom v1.4.0/go.mod h1:k/zktXdL+qnA6OgKsdEGUTA17jbQ2ZPTUa3CCySuGpE=
github.com/twpayne/go-kml v1.5.2/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4=
github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@ -976,11 +1001,11 @@ go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIj
go.opentelemetry.io/otel/exporters/jaeger v1.13.0 h1:VAMoGujbVV8Q0JNM/cEbhzUIWWBxnEqH45HP9iBKN04=
go.opentelemetry.io/otel/exporters/jaeger v1.13.0/go.mod h1:fHwbmle6mBFJA1p2ZIhilvffCdq/dM5UTIiCOmEjS+w=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0 h1:R3X6ZXmNPRR8ul6i3WgFURCHzaXjHdm0karRG/+dj3s=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.28.0/go.mod h1:QWFXnDavXWwMx2EEcZsf3yxgEKAqsxQ+Syjp+seyInw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.20.0 h1:CsBiKCiQPdSjS+MlRiqeTI9JDDpSuk0Hb6QTRfwer8k=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.20.0/go.mod h1:CMJYNAfooOwSZSAmAeMUV1M+TXld3BiK++z9fqIm2xk=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0 h1:4s9HxB4azeeQkhY0GE5wZlMj4/pz8tE5gx2OQpGUw58=
@ -995,8 +1020,8 @@ go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+
go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg=
go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I=
go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
@ -1032,6 +1057,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@ -1109,6 +1135,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -1196,6 +1223,7 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -1283,6 +1311,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1424,8 +1453,8 @@ google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+n
google.golang.org/genproto v0.0.0-20220503193339-ba3ae3f07e29/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d h1:PksQg4dV6Sem3/HkBX+Ltq8T0ke0PKIRBNBatoDTVls=
google.golang.org/genproto v0.0.0-20240624140628-dc46fd24d27d/go.mod h1:s7iA721uChleev562UJO2OYB0PPT9CMFjV+Ce7VJH5M=
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4 h1:MuYw1wJzT+ZkybKfaOXKp5hJiZDn2iHaXRw0mRYdHSc=
google.golang.org/genproto/googleapis/api v0.0.0-20240617180043-68d350f18fd4/go.mod h1:px9SlOOZBg1wM1zdnr8jEL4CNGUBZ+ZKYtNPApNQc4c=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094 h1:0+ozOGcrp+Y8Aq8TLNN2Aliibms5LEzsq99ZZmAGYm0=
google.golang.org/genproto/googleapis/api v0.0.0-20240701130421-f6361c86f094/go.mod h1:fJ/e3If/Q67Mj99hin0hMhiNyCRmt6BQ2aWIJshUSJw=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf h1:liao9UHurZLtiEwBgT9LMOnKYsHze6eA6w1KQCMVN2Q=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240730163845-b1a4ccb954bf/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
@ -1523,6 +1552,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

4
internal/core/conanfile.py
View File

@ -44,7 +44,8 @@ class MilvusConan(ConanFile):
"abseil/20230125.3#dad7cc4c83bbd44c1f1cc9cc4d97ac88",
"roaring/3.0.0#25a703f80eda0764a31ef939229e202d",
"grpc/1.50.1@milvus/dev#75103960d1cac300cf425ccfccceac08",
"rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c"
"rapidjson/cci.20230929#624c0094d741e6a3749d2e44d834b96c",
"gdal/3.5.3#0d131926ca2d68ab6c132e045a0e097d"
)
generators = ("cmake", "cmake_find_package")
default_options = {
@ -77,7 +78,6 @@ class MilvusConan(ConanFile):
"prometheus-cpp:with_pull": False,
"fmt:header_only": True,
"onetbb:tbbmalloc": False,
"onetbb:tbbproxy": False,
}
def configure(self):

24
internal/core/src/common/Array.h
View File

@ -218,7 +218,9 @@ class Array {
return true;
}
case DataType::STRING:
case DataType::VARCHAR: {
case DataType::VARCHAR:
//treat Geometry as wkb string
case DataType::GEOMETRY: {
for (int i = 0; i < length_; ++i) {
if (get_data<std::string_view>(i) !=
arr.get_data<std::string_view>(i)) {
@ -331,6 +333,13 @@ class Array {
}
break;
}
case DataType::GEOMETRY: {
for (int j = 0; j < length_; ++j) {
auto element = get_data<std::string>(j);
data_array.mutable_geometry_data()->add_data(element);
}
break;
}
default: {
// empty array
}
@ -418,7 +427,8 @@ class Array {
return true;
}
case DataType::VARCHAR:
case DataType::STRING: {
case DataType::STRING:
case DataType::GEOMETRY: {
for (int i = 0; i < length_; i++) {
auto val = get_data<std::string>(i);
if (val != arr2.array(i).string_val()) {
@ -554,6 +564,13 @@ class ArrayView {
}
break;
}
case DataType::GEOMETRY: {
for (int j = 0; j < length_; ++j) {
auto element = get_data<std::string>(j);
data_array.mutable_geometry_data()->add_data(element);
}
break;
}
default: {
// empty array
}
@ -643,7 +660,8 @@ class ArrayView {
return true;
}
case DataType::VARCHAR:
case DataType::STRING: {
case DataType::STRING:
case DataType::GEOMETRY: {
for (int i = 0; i < length_; i++) {
auto val = get_data<std::string>(i);
if (val != arr2.array(i).string_val()) {

20
internal/core/src/common/FieldData.cpp
View File

@ -212,6 +212,22 @@ FieldDataImpl<Type, is_type_entire_row>::FillFieldData(
}
return FillFieldData(values.data(), element_count);
}
case DataType::GEOMETRY: {
auto array_info =
GetDataInfoFromArray<arrow::BinaryViewArray,
arrow::Type::type::BINARY>(array);
auto geometry_array =
std::dynamic_pointer_cast<arrow::BinaryArray>(array);
std::vector<std::string> values(element_count);
for (size_t index = 0; index < element_count; ++index) {
values[index] = geometry_array->GetString(index);
}
if (nullable_) {
return FillFieldData(
values.data(), array->null_bitmap_data(), element_count);
}
return FillFieldData(values.data(), array_info.second);
}
case DataType::ARRAY: {
auto array_array =
std::dynamic_pointer_cast<arrow::BinaryArray>(array);
@ -277,6 +293,7 @@ template class FieldDataImpl<float, true>;
template class FieldDataImpl<double, true>;
template class FieldDataImpl<std::string, true>;
template class FieldDataImpl<Json, true>;
template class FieldDataImpl<Geometry, true>;
template class FieldDataImpl<Array, true>;
// vector data
@ -314,6 +331,9 @@ InitScalarFieldData(const DataType& type, bool nullable, int64_t cap_rows) {
type, nullable, cap_rows);
case DataType::JSON:
return std::make_shared<FieldData<Json>>(type, nullable, cap_rows);
case DataType::GEOMETRY:
return std::make_shared<FieldData<Geometry>>(
type, nullable, cap_rows);
default:
PanicInfo(DataTypeInvalid,
"InitScalarFieldData not support data type " +

11
internal/core/src/common/FieldData.h
View File

@ -68,6 +68,17 @@ class FieldData<Json> : public FieldDataJsonImpl {
}
};
template <>
class FieldData<Geometry> : public FieldDataGeometryImpl {
public:
static_assert(IsScalar<Geometry> || std::is_same_v<std::string, PkType>);
explicit FieldData(DataType data_type,
bool nullable,
int64_t buffered_num_rows = 0)
: FieldDataGeometryImpl(data_type, nullable, buffered_num_rows) {
}
};
template <>
class FieldData<Array> : public FieldDataArrayImpl {
public:

62
internal/core/src/common/FieldDataInterface.h
View File

@ -573,6 +573,68 @@ class FieldDataStringImpl : public FieldDataImpl<std::string, true> {
}
};
class FieldDataGeometryImpl : public FieldDataImpl<std::string, true> {
public:
explicit FieldDataGeometryImpl(DataType data_type,
bool nullable,
int64_t total_num_rows = 0)
: FieldDataImpl<std::string, true>(
1, data_type, nullable, total_num_rows) {
}
int64_t
DataSize() const override {
int64_t data_size = 0;
for (size_t offset = 0; offset < length(); ++offset) {
data_size += data_[offset].size();
}
return data_size;
}
int64_t
DataSize(ssize_t offset) const override {
AssertInfo(offset < get_num_rows(),
"field data subscript out of range");
AssertInfo(offset < length(),
"subscript position don't has valid value");
return data_[offset].size();
}
void
FillFieldData(const std::shared_ptr<arrow::Array> array) override {
AssertInfo(array->type()->id() == arrow::Type::type::BINARY,
"inconsistent data type, expected: {}, got: {}",
"BINARY",
array->type()->ToString());
auto geometry_array =
std::dynamic_pointer_cast<arrow::BinaryArray>(array);
FillFieldData(geometry_array);
}
void
FillFieldData(const std::shared_ptr<arrow::BinaryArray>& array) override {
auto n = array->length();
if (n == 0) {
return;
}
null_count_ = array->null_count();
std::lock_guard lck(tell_mutex_);
if (length_ + n > get_num_rows()) {
resize_field_data(length_ + n);
}
auto i = 0;
for (const auto& geometry : *array) {
if (!geometry.has_value()) {
i++;
continue;
}
data_[length_ + i] = geometry.value();
i++;
}
length_ += n;
}
};
class FieldDataJsonImpl : public FieldDataImpl<Json, true> {
public:
explicit FieldDataJsonImpl(DataType data_type,

176
internal/core/src/common/Geometry.h Normal file
View File

@ -0,0 +1,176 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include <gdal.h>
#include <ogr_geometry.h>
#include <algorithm>
#include <memory>
#include <string_view>
#include "common/EasyAssert.h"
namespace milvus {
class Geometry {
public:
Geometry() = default;
// all ctr assume that wkb data is valid
explicit Geometry(const void* wkb, size_t size) {
OGRGeometry* geometry = nullptr;
OGRGeometryFactory::createFromWkb(wkb, nullptr, &geometry, size);
AssertInfo(geometry != nullptr,
"failed to construct geometry from wkb data");
geometry_.reset(geometry);
to_wkb_internal();
}
explicit Geometry(const char* wkt) {
OGRGeometry* geometry = nullptr;
OGRGeometryFactory::createFromWkt(wkt, nullptr, &geometry);
AssertInfo(geometry != nullptr,
"failed to construct geometry from wkt data");
geometry_.reset(geometry);
to_wkb_internal();
}
Geometry(const Geometry& other) {
if (other.IsValid()) {
this->geometry_.reset(other.geometry_->clone());
this->to_wkb_internal();
}
}
Geometry(Geometry&& other) noexcept
: wkb_data_(std::move(other.wkb_data_)),
size_(std::move(other.size_)),
geometry_(std::move(other.geometry_)) {
}
Geometry&
operator=(const Geometry& other) {
if (this != &other && other.IsValid()) {
this->geometry_.reset(other.geometry_->clone());
this->to_wkb_internal();
}
return *this;
}
Geometry&
operator=(Geometry&& other) noexcept {
if (this != &other) {
wkb_data_ = std::move(other.wkb_data_);
size_ = std::move(other.size_);
geometry_ = std::move(other.geometry_);
}
return *this;
}
operator std::string() const {
//tmp string created by copy ctr
return std::string(reinterpret_cast<const char*>(wkb_data_.get()));
}
operator std::string_view() const {
return std::string_view(reinterpret_cast<const char*>(wkb_data_.get()),
size_);
}
~Geometry(){
if(geometry_){
OGRGeometryFactory::destroyGeometry(geometry_.release());
}
}
bool
IsValid() const {
return geometry_ != nullptr;
}
OGRGeometry*
GetGeometry() const {
return geometry_.get();
}
const unsigned char*
data() const {
return wkb_data_.get();
}
size_t
size() const {
return size_;
}
//spatial relation
bool
equals(const Geometry& other) const {
return geometry_->Equals(other.geometry_.get());
}
bool
touches(const Geometry& other) const {
return geometry_->Touches(other.geometry_.get());
}
bool
overlaps(const Geometry& other) const {
return geometry_->Overlaps(other.geometry_.get());
}
bool
crosses(const Geometry& other) const {
return geometry_->Crosses(other.geometry_.get());
}
bool
contains(const Geometry& other) const {
return geometry_->Contains(other.geometry_.get());
}
bool
intersects(const Geometry& other) const {
return geometry_->Intersects(other.geometry_.get());
}
bool
within(const Geometry& other) const {
return geometry_->Within(other.geometry_.get());
}
std::string
to_wkt_string() const {
return geometry_->exportToWkt();
}
std::string
to_wkb_string() const {
return std::string(reinterpret_cast<char*>(wkb_data_.get()), size_);
}
private:
inline void
to_wkb_internal() {
if (geometry_) {
size_ = geometry_->WkbSize();
wkb_data_ = std::make_unique<unsigned char[]>(size_);
// little-endian order to save wkb
geometry_->exportToWkb(wkbNDR, wkb_data_.get());
}
}
std::unique_ptr<unsigned char[]> wkb_data_;
size_t size_{0};
std::unique_ptr<OGRGeometry> geometry_;
};
} // namespace milvus

30
internal/core/src/common/Types.h
View File

@ -45,6 +45,7 @@
#include "pb/schema.pb.h"
#include "pb/segcore.pb.h"
#include "Json.h"
#include "Geometry.h"
#include "CustomBitset.h"
@ -75,6 +76,7 @@ enum class DataType {
VARCHAR = 21,
ARRAY = 22,
JSON = 23,
GEOMETRY = 24,
// Some special Data type, start from after 50
// just for internal use now, may sync proto in future
@ -180,6 +182,8 @@ GetDataTypeName(DataType data_type) {
return "array";
case DataType::JSON:
return "json";
case DataType::GEOMETRY:
return "geometry";
case DataType::VECTOR_FLOAT:
return "vector_float";
case DataType::VECTOR_BINARY:
@ -215,6 +219,7 @@ using GroupByValueType = std::variant<std::monostate,
bool,
std::string>;
using ContainsType = proto::plan::JSONContainsExpr_JSONOp;
using GISFunctionType = proto::plan::GISFunctionFilterExpr_GISOp;
inline bool
IsPrimaryKeyDataType(DataType data_type) {
@ -261,6 +266,11 @@ IsJsonDataType(DataType data_type) {
return data_type == DataType::JSON;
}
inline bool
IsGeometryDataType(DataType data_type) {
return data_type == DataType::GEOMETRY;
}
inline bool
IsArrayDataType(DataType data_type) {
return data_type == DataType::ARRAY;
@ -268,7 +278,8 @@ IsArrayDataType(DataType data_type) {
inline bool
IsBinaryDataType(DataType data_type) {
return IsJsonDataType(data_type) || IsArrayDataType(data_type);
return IsJsonDataType(data_type) || IsArrayDataType(data_type) ||
IsGeometryDataType(data_type);
}
inline bool
@ -294,6 +305,11 @@ IsJsonType(proto::schema::DataType type) {
return type == proto::schema::DataType::JSON;
}
inline bool
IsGeometryType(proto::schema::DataType type) {
return type == proto::schema::DataType::Geometry;
}
inline bool
IsArrayType(proto::schema::DataType type) {
return type == proto::schema::DataType::Array;
@ -536,6 +552,15 @@ struct TypeTraits<DataType::JSON> {
static constexpr const char* Name = "JSON";
};
template <>
struct TypeTraits<DataType::GEOMETRY> {
using NativeType = void;
static constexpr DataType TypeKind = DataType::GEOMETRY;
static constexpr bool IsPrimitiveType = false;
static constexpr bool IsFixedWidth = false;
static constexpr const char* Name = "GEOMETRY";
};
template <>
struct TypeTraits<DataType::ROW> {
using NativeType = void;
@ -606,6 +631,9 @@ struct fmt::formatter<milvus::DataType> : formatter<string_view> {
case milvus::DataType::JSON:
name = "JSON";
break;
case milvus::DataType::GEOMETRY:
name = "GEOMETRY";
break;
case milvus::DataType::ROW:
name = "ROW";
break;

8
internal/core/src/common/VectorTrait.h
View File

@ -60,9 +60,9 @@ constexpr bool IsVector = std::is_base_of_v<VectorTrait, T>;
template <typename T>
constexpr bool IsScalar =
std::is_fundamental_v<T> || std::is_same_v<T, std::string> ||
std::is_same_v<T, Json> || std::is_same_v<T, std::string_view> ||
std::is_same_v<T, Array> || std::is_same_v<T, ArrayView> ||
std::is_same_v<T, proto::plan::Array>;
std::is_same_v<T, Json> || std::is_same_v<T, Geometry> ||
std::is_same_v<T, std::string_view> || std::is_same_v<T, Array> ||
std::is_same_v<T, ArrayView> || std::is_same_v<T, proto::plan::Array>;
template <typename T>
constexpr bool IsSparse = std::is_same_v<T, SparseFloatVector> ||
@ -73,7 +73,7 @@ constexpr bool IsVariableType =
std::is_same_v<T, std::string> || std::is_same_v<T, std::string_view> ||
std::is_same_v<T, Array> || std::is_same_v<T, ArrayView> ||
std::is_same_v<T, proto::plan::Array> || std::is_same_v<T, Json> ||
IsSparse<T>;
std::is_same_v<T, Geometry> || IsSparse<T>;
template <typename T>
constexpr bool IsVariableTypeSupportInChunk =

13
internal/core/src/exec/expression/Expr.cpp
View File

@ -25,12 +25,15 @@
#include "exec/expression/CompareExpr.h"
#include "exec/expression/ConjunctExpr.h"
#include "exec/expression/ExistsExpr.h"
#include "exec/expression/GISFunctionFilterExpr.h"
#include "exec/expression/JsonContainsExpr.h"
#include "exec/expression/LogicalBinaryExpr.h"
#include "exec/expression/LogicalUnaryExpr.h"
#include "exec/expression/TermExpr.h"
#include "exec/expression/UnaryExpr.h"
#include "expr/ITypeExpr.h"
#include "exec/expression/ValueExpr.h"
#include "log/Log.h"
#include <memory>
@ -285,6 +288,16 @@ CompileExpression(const expr::TypedExprPtr& expr,
context->get_segment(),
context->get_active_count(),
context->query_config()->get_expr_batch_size());
} else if (auto casted_expr = std::dynamic_pointer_cast<
const milvus::expr::GISFunctioinFilterExpr>(expr)) {
LOG_WARN("trans to phygisexpr!!");
result = std::make_shared<PhyGISFunctionFilterExpr>(
compiled_inputs,
casted_expr,
"PhyGISFunctionFilterExpr",
context->get_segment(),
context->get_active_count(),
context->query_config()->get_expr_batch_size());
} else {
PanicInfo(ExprInvalid, "unsupport expr: ", expr->ToString());
}

108
internal/core/src/exec/expression/GISFunctionFilterExpr.cpp Normal file
View File

@ -0,0 +1,108 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "GISFunctionFilterExpr.h"
#include "common/EasyAssert.h"
#include "common/Geometry.h"
#include "common/Types.h"
#include "pb/plan.pb.h"
namespace milvus {
namespace exec {
#define GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(method) \
auto execute_sub_batch = [](const std::string_view* data, \
const bool* valid_data, \
const int size, \
TargetBitmapView res, \
TargetBitmapView valid_res, \
const Geometry& right_source) { \
for (int i = 0; i < size; ++i) { \
if (valid_data != nullptr && !valid_data[i]) { \
res[i] = valid_res[i] = false; \
continue; \
} \
res[i] = \
Geometry(data[i].data(), data[i].size()).method(right_source); \
} \
}; \
int64_t processed_size = ProcessDataChunks<std::string_view>( \
execute_sub_batch, std::nullptr_t{}, res, valid_res, right_source); \
AssertInfo(processed_size == real_batch_size, \
"internal error: expr processed rows {} not equal " \
"expect batch size {}", \
processed_size, \
real_batch_size); \
return res_vec;
void
PhyGISFunctionFilterExpr::Eval(EvalCtx& context, VectorPtr& result) {
AssertInfo(expr_->column_.data_type_ == DataType::GEOMETRY,
"unsupported data type: {}",
expr_->column_.data_type_);
if (is_index_mode_) {
// result = EvalForIndexSegment();
PanicInfo(NotImplemented, "index for geos not implement");
} else {
result = EvalForDataSegment();
}
}
VectorPtr
PhyGISFunctionFilterExpr::EvalForDataSegment() {
auto real_batch_size = GetNextBatchSize();
if (real_batch_size == 0) {
return nullptr;
}
auto res_vec = std::make_shared<ColumnVector>(
TargetBitmap(real_batch_size), TargetBitmap(real_batch_size));
TargetBitmapView res(res_vec->GetRawData(), real_batch_size);
TargetBitmapView valid_res(res_vec->GetValidRawData(), real_batch_size);
valid_res.set();
auto& right_source = expr_->geometry_;
switch (expr_->op_) {
case proto::plan::GISFunctionFilterExpr_GISOp_Equals: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(equals);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Touches: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(touches);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Overlaps: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(overlaps);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Crosses: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(crosses);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Contains: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(contains);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Intersects: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(intersects);
}
case proto::plan::GISFunctionFilterExpr_GISOp_Within: {
GEOMETRY_EXECUTE_SUB_BATCH_WITH_COMPARISON(within);
}
default: {
PanicInfo(NotImplemented,
"internal error: unknown GIS op : {}",
expr_->op_);
}
}
return res_vec;
}
// VectorPtr
// PhyGISFunctionFilterExpr::EvalForIndexSegment() {
// // TODO
// }
} //namespace exec
} // namespace milvus

56
internal/core/src/exec/expression/GISFunctionFilterExpr.h Normal file
View File

@ -0,0 +1,56 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include <fmt/core.h>
#include "common/Vector.h"
#include "exec/expression/Expr.h"
#include "expr/ITypeExpr.h"
#include "segcore/SegmentInterface.h"
namespace milvus {
namespace exec {
class PhyGISFunctionFilterExpr : public SegmentExpr {
public:
PhyGISFunctionFilterExpr(
const std::vector<std::shared_ptr<Expr>>& input,
const std::shared_ptr<const milvus::expr::GISFunctioinFilterExpr>& expr,
const std::string& name,
const segcore::SegmentInternalInterface* segment,
int64_t active_count,
int64_t batch_size)
: SegmentExpr(std::move(input),
name,
segment,
expr->column_.field_id_,
active_count,
batch_size),
expr_(expr) {
}
void
Eval(EvalCtx& context, VectorPtr& result) override;
private:
// VectorPtr
// EvalForIndexSegment();
VectorPtr
EvalForDataSegment();
private:
std::shared_ptr<const milvus::expr::GISFunctioinFilterExpr> expr_;
};
} //namespace exec
} // namespace milvus

24
internal/core/src/expr/ITypeExpr.h
View File

@ -21,11 +21,13 @@
#include <string>
#include <vector>
#include "common/Geometry.h"
#include "exec/expression/function/FunctionFactory.h"
#include "common/Exception.h"
#include "common/Schema.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "ogr_geometry.h"
#include "pb/plan.pb.h"
namespace milvus {
@ -705,6 +707,28 @@ class CompareExpr : public ITypeFilterExpr {
const proto::plan::OpType op_type_;
};
class GISFunctioinFilterExpr : public ITypeFilterExpr {
public:
GISFunctioinFilterExpr(ColumnInfo cloumn,
GISFunctionType op,
const Geometry& geometry)
: column_(cloumn), op_(op), geometry_(geometry){};
std::string
ToString() const override {
return fmt::format(
"GISFunctioinFilterExpr:[Column: {}, Operator: {} "
"WktValue: {}]",
column_.ToString(),
GISFunctionFilterExpr_GISOp_Name(op_),
geometry_.to_wkt_string());
}
public:
const ColumnInfo column_;
const GISFunctionType op_;
const Geometry geometry_;
};
class JsonContainsExpr : public ITypeFilterExpr {
public:
JsonContainsExpr(ColumnInfo column,

22
internal/core/src/mmap/Utils.h
View File

@ -26,6 +26,7 @@
#include <vector>
#include "common/FieldMeta.h"
#include "common/Geometry.h"
#include "common/Types.h"
#include "mmap/Types.h"
#include "storage/Util.h"
@ -136,6 +137,27 @@ WriteFieldData(File& file,
}
break;
}
case DataType::GEOMETRY: {
// write as: |size|data|size|data......
for (ssize_t i = 0; i < data->get_num_rows(); ++i) {
indices.push_back(total_written);
auto geo_ptr =
static_cast<const std::string*>(data->RawValue(i));
ssize_t written_data_size =
file.WriteInt<uint32_t>(geo_ptr->size());
if (written_data_size != sizeof(uint32_t)) {
THROW_FILE_WRITE_ERROR
}
total_written += written_data_size;
ssize_t written_data =
file.Write(geo_ptr->data(), geo_ptr->size());
if (written_data < geo_ptr->size()) {
THROW_FILE_WRITE_ERROR
}
total_written += written_data;
}
break;
}
case DataType::ARRAY: {
// write as: |data|data|data|data|data......
for (size_t i = 0; i < data->get_num_rows(); ++i) {

25
internal/core/src/query/PlanProto.cpp
View File

@ -14,12 +14,17 @@
#include <google/protobuf/text_format.h>
#include <cstdint>
#include <memory>
#include <string>
#include <vector>
#include "common/Geometry.h"
#include "common/VectorTrait.h"
#include "common/EasyAssert.h"
#include "exec/expression/function/FunctionFactory.h"
#include "log/Log.h"
#include "ogr_core.h"
#include "ogr_geometry.h"
#include "pb/plan.pb.h"
#include "query/Utils.h"
#include "knowhere/comp/materialized_view.h"
@ -384,6 +389,20 @@ ProtoParser::ParseValueExprs(const proto::plan::ValueExpr& expr_pb) {
return std::make_shared<expr::ValueExpr>(expr_pb.value());
}
expr::TypedExprPtr
ProtoParser::ParseGISFunctionFilterExprs(
const proto::plan::GISFunctionFilterExpr& expr_pb) {
auto& columnInfo = expr_pb.column_info();
auto field_id = FieldId(columnInfo.field_id());
auto data_type = schema[field_id].get_data_type();
Assert(data_type == (DataType)columnInfo.data_type());
const std::string& str = expr_pb.wkt_string();
Geometry geometry(str.data());
LOG_WARN("{}", geometry.to_wkt_string());
return std::make_shared<expr::GISFunctioinFilterExpr>(
columnInfo, expr_pb.op(), geometry);
}
expr::TypedExprPtr
ProtoParser::CreateAlwaysTrueExprs() {
return std::make_shared<expr::AlwaysTrueExpr>();
@ -449,6 +468,11 @@ ProtoParser::ParseExprs(const proto::plan::Expr& expr_pb,
result = ParseValueExprs(expr_pb.value_expr());
break;
}
case ppe::kGisfunctionFilterExpr: {
result =
ParseGISFunctionFilterExprs(expr_pb.gisfunction_filter_expr());
break;
}
default: {
std::string s;
google::protobuf::TextFormat::PrintToString(expr_pb, &s);
@ -457,6 +481,7 @@ ProtoParser::ParseExprs(const proto::plan::Expr& expr_pb,
}
}
if (type_check(result->type())) {
LOG_WARN("passed check");
return result;
}
PanicInfo(

4
internal/core/src/query/PlanProto.h
View File

@ -83,6 +83,10 @@ class ProtoParser {
expr::TypedExprPtr
ParseJsonContainsExprs(const proto::plan::JSONContainsExpr& expr_pb);
expr::TypedExprPtr
ParseGISFunctionFilterExprs(
const proto::plan::GISFunctionFilterExpr& expr_pb);
expr::TypedExprPtr
ParseTermExprs(const proto::plan::TermExpr& expr_pb);

11
internal/core/src/segcore/ConcurrentVector.cpp
View File

@ -98,6 +98,17 @@ VectorBase::set_data_raw(ssize_t element_offset,
return set_data_raw(element_offset, data_raw.data(), element_count);
}
case DataType::GEOMETRY: {
// get the geometry array of a column from proto message
auto& geometry_data = FIELD_DATA(data, geometry);
std::vector<std::string> data_raw{};
data_raw.reserve(geometry_data.size());
for (auto& geometry_bytes : geometry_data) {
//this geometry_bytes consider as wkt strings from milvus-proto
data_raw.emplace_back(std::string(geometry_bytes));
}
return set_data_raw(element_offset, data_raw.data(), element_count);
}
case DataType::ARRAY: {
auto& array_data = FIELD_DATA(data, array);
std::vector<Array> data_raw{};

4
internal/core/src/segcore/InsertRecord.h
View File

@ -455,6 +455,10 @@ struct InsertRecord {
this->append_data<Json>(field_id, size_per_chunk);
break;
}
case DataType::GEOMETRY: {
this->append_data<std::string>(field_id, size_per_chunk);
break;
}
case DataType::ARRAY: {
this->append_data<Array>(field_id, size_per_chunk);
break;

10
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <algorithm>
#include <cstdlib>
#include <cstring>
#include <memory>
#include <numeric>
@ -607,6 +608,15 @@ SegmentGrowingImpl::bulk_subscript(FieldId field_id,
result->mutable_scalars()->mutable_json_data()->mutable_data());
break;
}
case DataType::GEOMETRY: {
bulk_subscript_ptr_impl<std::string>(vec_ptr,
seg_offsets,
count,
result->mutable_scalars()
->mutable_geometry_data()
->mutable_data());
break;
}
case DataType::ARRAY: {
// element
bulk_subscript_array_impl(*vec_ptr,

35
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -35,6 +35,7 @@
#include "common/FieldMeta.h"
#include "common/File.h"
#include "common/Json.h"
#include "common/Geometry.h"
#include "common/LoadInfo.h"
#include "common/Tracer.h"
#include "common/Types.h"
@ -416,6 +417,20 @@ SegmentSealedImpl::LoadFieldData(FieldId field_id, FieldDataInfo& data) {
column = std::move(var_column);
break;
}
case milvus::DataType::GEOMETRY: {
auto var_column = std::make_shared<
SingleChunkVariableColumn<std::string>>(
num_rows, field_meta, get_block_size());
FieldDataPtr field_data;
while (data.channel->pop(field_data)) {
var_column->Append(std::move(field_data));
}
var_column->Seal();
stats_.mem_size += var_column->MemoryUsageBytes();
field_data_size = var_column->DataByteSize();
column = std::move(var_column);
break;
}
case milvus::DataType::ARRAY: {
auto var_column = std::make_shared<SingleChunkArrayColumn>(
num_rows, field_meta);
@ -585,6 +600,17 @@ SegmentSealedImpl::MapFieldData(const FieldId field_id, FieldDataInfo& data) {
column = std::move(var_column);
break;
}
case milvus::DataType::GEOMETRY: {
auto var_column =
std::make_shared<SingleChunkVariableColumn<std::string>>(
file,
total_written,
field_meta,
DEFAULT_MMAP_VRCOL_BLOCK_SIZE);
var_column->Seal(std::move(indices));
column = std::move(var_column);
break;
}
case milvus::DataType::ARRAY: {
auto arr_column = std::make_shared<SingleChunkArrayColumn>(
file, total_written, field_meta);
@ -1474,6 +1500,15 @@ SegmentSealedImpl::get_raw_data(FieldId field_id,
break;
}
case DataType::GEOMETRY: {
bulk_subscript_ptr_impl<std::string>(column.get(),
seg_offsets,
count,
ret->mutable_scalars()
->mutable_geometry_data()
->mutable_data());
break;
}
case DataType::ARRAY: {
bulk_subscript_array_impl(
column.get(),

31
internal/core/src/segcore/Utils.cpp
View File

@ -19,7 +19,9 @@
#include "common/Common.h"
#include "common/FieldData.h"
#include "common/FieldDataInterface.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "index/ScalarIndex.h"
#include "mmap/Utils.h"
#include "log/Log.h"
@ -146,6 +148,13 @@ GetRawDataSizeOfDataArray(const DataArray* data,
}
break;
}
case DataType::GEOMETRY: {
auto& geometry_data = FIELD_DATA(data, geometry);
for (auto& geometry_bytes : geometry_data) {
result += geometry_bytes.size();
}
break;
}
case DataType::ARRAY: {
auto& array_data = FIELD_DATA(data, array);
switch (field_meta.get_element_type()) {
@ -292,6 +301,14 @@ CreateScalarDataArray(int64_t count, const FieldMeta& field_meta) {
}
break;
}
case DataType::GEOMETRY: {
auto obj = scalar_array->mutable_geometry_data();
obj->mutable_data()->Reserve(count);
for (int i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = std::string();
}
break;
}
case DataType::ARRAY: {
auto obj = scalar_array->mutable_array_data();
obj->mutable_data()->Reserve(count);
@ -440,6 +457,14 @@ CreateScalarDataArrayFrom(const void* data_raw,
}
break;
}
case DataType::GEOMETRY: {
auto data = reinterpret_cast<const std::string*>(data_raw);
auto obj = scalar_array->mutable_geometry_data();
for (auto i = 0; i < count; i++) {
*(obj->mutable_data()->Add()) = data[i];
}
break;
}
case DataType::ARRAY: {
auto data = reinterpret_cast<const ScalarArray*>(data_raw);
auto obj = scalar_array->mutable_array_data();
@ -655,6 +680,12 @@ MergeDataArray(std::vector<MergeBase>& merge_bases,
*(obj->mutable_data()->Add()) = data[src_offset];
break;
}
case DataType::GEOMETRY: {
auto& data = FIELD_DATA(src_field_data, geometry);
auto obj = scalar_array->mutable_geometry_data();
*(obj->mutable_data()->Add()) = data[src_offset];
break;
}
case DataType::ARRAY: {
auto& data = FIELD_DATA(src_field_data, array);
auto obj = scalar_array->mutable_array_data();

12
internal/core/src/storage/Event.cpp
View File

@ -21,6 +21,7 @@
#include "common/Consts.h"
#include "common/EasyAssert.h"
#include "common/FieldMeta.h"
#include "common/Geometry.h"
#include "common/Json.h"
#include "fmt/format.h"
#include "nlohmann/json.hpp"
@ -282,6 +283,17 @@ BaseEventData::Serialize() {
}
break;
}
case DataType::GEOMETRY: {
for (size_t offset = 0; offset < field_data->get_num_rows();
++offset) {
auto geo_ptr = static_cast<const std::string*>(
field_data->RawValue(offset));
payload_writer->add_one_binary_payload(
reinterpret_cast<const uint8_t*>(geo_ptr->data()),
geo_ptr->size());
}
break;
}
case DataType::VECTOR_SPARSE_FLOAT: {
for (size_t offset = 0; offset < field_data->get_num_rows();
++offset) {

9
internal/core/src/storage/Util.cpp
View File

@ -275,7 +275,8 @@ CreateArrowBuilder(DataType data_type) {
return std::make_shared<arrow::StringBuilder>();
}
case DataType::ARRAY:
case DataType::JSON: {
case DataType::JSON:
case DataType::GEOMETRY: {
return std::make_shared<arrow::BinaryBuilder>();
}
// sparse float vector doesn't require a dim
@ -356,7 +357,8 @@ CreateArrowSchema(DataType data_type, bool nullable) {
{arrow::field("val", arrow::utf8(), nullable)});
}
case DataType::ARRAY:
case DataType::JSON: {
case DataType::JSON:
case DataType::GEOMETRY: {
return arrow::schema(
{arrow::field("val", arrow::binary(), nullable)});
}
@ -780,6 +782,9 @@ CreateFieldData(const DataType& type,
case DataType::JSON:
return std::make_shared<FieldData<Json>>(
type, nullable, total_num_rows);
case DataType::GEOMETRY:
return std::make_shared<FieldData<Geometry>>(
type, nullable, total_num_rows);
case DataType::ARRAY:
return std::make_shared<FieldData<Array>>(
type, nullable, total_num_rows);

5
internal/core/unittest/test_chunk_vector.cpp
View File

@ -51,6 +51,7 @@ TEST_F(ChunkVectorTest, FillDataWithMmap) {
auto double_field = schema->AddDebugField("double", DataType::DOUBLE);
auto varchar_field = schema->AddDebugField("varchar", DataType::VARCHAR);
auto json_field = schema->AddDebugField("json", DataType::JSON);
auto geometry_field = schema->AddDebugField("geometry", DataType::GEOMETRY);
auto int_array_field =
schema->AddDebugField("int_array", DataType::ARRAY, DataType::INT8);
auto long_array_field =
@ -116,6 +117,8 @@ TEST_F(ChunkVectorTest, FillDataWithMmap) {
varchar_field, ids_ds->GetIds(), num_inserted);
auto json_result =
segment->bulk_subscript(json_field, ids_ds->GetIds(), num_inserted);
auto geometry_result = segment->bulk_subscript(
geometry_field, ids_ds->GetIds(), num_inserted);
auto int_array_result = segment->bulk_subscript(
int_array_field, ids_ds->GetIds(), num_inserted);
auto long_array_result = segment->bulk_subscript(
@ -150,6 +153,8 @@ TEST_F(ChunkVectorTest, FillDataWithMmap) {
EXPECT_EQ(varchar_result->scalars().string_data().data_size(),
num_inserted);
EXPECT_EQ(json_result->scalars().json_data().data_size(), num_inserted);
EXPECT_EQ(geometry_result->scalars().geometry_data().data_size(),
num_inserted);
EXPECT_EQ(fp32_vec_result->vectors().float_vector().data_size(),
num_inserted * dim);
EXPECT_EQ(fp16_vec_result->vectors().float16_vector().size(),

57
internal/core/unittest/test_data_codec.cpp
View File

@ -17,6 +17,8 @@
#include <gtest/gtest.h>
#include <string>
#include "ogr_core.h"
#include "ogr_geometry.h"
#include "storage/DataCodec.h"
#include "storage/InsertData.h"
#include "storage/IndexData.h"
@ -331,7 +333,62 @@ TEST(storage, InsertDataInt64Nullable) {
ASSERT_EQ(*new_payload->ValidData(), *valid_data);
delete[] valid_data;
}
TEST(storage, InsertDataGeometry) {
OGRPoint point1(10.25, 0.55), point2(9.75, -0.23), point3(-8.50, 1.44);
OGRLineString linstring;
linstring.addPoint(&point1);
linstring.addPoint(&point2);
linstring.addPoint(&point3);
OGRPolygon polygon;
OGRLinearRing ring;
ring.addPoint(&point1);
ring.addPoint(&point2);
ring.addPoint(&point3);
ring.closeRings();
polygon.addRing(&ring);
std::string str1, str2, str3;
unsigned char *s1 = new unsigned char[point1.WkbSize()],
*s2 = new unsigned char[linstring.WkbSize()],
*s3 = new unsigned char[polygon.WkbSize()];
point1.exportToWkb(wkbNDR, s1);
linstring.exportToWkb(wkbNDR, s2);
polygon.exportToWkb(wkbNDR, s3);
str1 = std::move(
std::string(reinterpret_cast<const char*>(s1), point1.WkbSize()));
str2 = std::move(
std::string(reinterpret_cast<const char*>(s2), linstring.WkbSize()));
str3 = std::move(
std::string(reinterpret_cast<const char*>(s3), polygon.WkbSize()));
FixedVector<std::string> data = {str1, str2, str3};
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::GEOMETRY, false);
field_data->FillFieldData(data.data(), data.size());
storage::InsertData insert_data(field_data);
storage::FieldDataMeta field_data_meta{100, 101, 102, 103};
insert_data.SetFieldDataMeta(field_data_meta);
insert_data.SetTimestamps(0, 100);
auto serialized_bytes = insert_data.Serialize(storage::StorageType::Remote);
std::shared_ptr<uint8_t[]> serialized_data_ptr(serialized_bytes.data(),
[&](uint8_t*) {});
auto new_insert_data = storage::DeserializeFileData(
serialized_data_ptr, serialized_bytes.size());
ASSERT_EQ(new_insert_data->GetCodecType(), storage::InsertDataType);
ASSERT_EQ(new_insert_data->GetTimeRage(),
std::make_pair(Timestamp(0), Timestamp(100)));
auto new_payload = new_insert_data->GetFieldData();
ASSERT_EQ(new_payload->get_data_type(), storage::DataType::GEOMETRY);
ASSERT_EQ(new_payload->get_num_rows(), data.size());
FixedVector<std::string> new_data(data.size());
ASSERT_EQ(new_payload->get_null_count(), 0);
for (int i = 0; i < data.size(); ++i) {
new_data[i] =
*static_cast<const std::string*>(new_payload->RawValue(i));
ASSERT_EQ(new_payload->DataSize(i), data[i].size());
}
ASSERT_EQ(data, new_data);
}
TEST(storage, InsertDataString) {
FixedVector<std::string> data = {
"test1", "test2", "test3", "test4", "test5"};

5
internal/core/unittest/test_growing.cpp
View File

@ -150,6 +150,7 @@ TEST_P(GrowingTest, FillData) {
auto double_field = schema->AddDebugField("double", DataType::DOUBLE);
auto varchar_field = schema->AddDebugField("varchar", DataType::VARCHAR);
auto json_field = schema->AddDebugField("json", DataType::JSON);
auto geometry_field = schema->AddDebugField("geometry", DataType::GEOMETRY);
auto int_array_field =
schema->AddDebugField("int_array", DataType::ARRAY, DataType::INT8);
auto long_array_field =
@ -213,6 +214,8 @@ TEST_P(GrowingTest, FillData) {
varchar_field, ids_ds->GetIds(), num_inserted);
auto json_result =
segment->bulk_subscript(json_field, ids_ds->GetIds(), num_inserted);
auto geometry_result = segment->bulk_subscript(
geometry_field, ids_ds->GetIds(), num_inserted);
auto int_array_result = segment->bulk_subscript(
int_array_field, ids_ds->GetIds(), num_inserted);
auto long_array_result = segment->bulk_subscript(
@ -241,6 +244,8 @@ TEST_P(GrowingTest, FillData) {
EXPECT_EQ(varchar_result->scalars().string_data().data_size(),
num_inserted);
EXPECT_EQ(json_result->scalars().json_data().data_size(), num_inserted);
EXPECT_EQ(geometry_result->scalars().geometry_data().data_size(),
num_inserted);
if (data_type == DataType::VECTOR_FLOAT) {
EXPECT_EQ(vec_result->vectors().float_vector().data_size(),
num_inserted * dim);

9
internal/core/unittest/test_sealed.cpp
View File

@ -407,6 +407,7 @@ TEST(Sealed, LoadFieldData) {
schema->AddDebugField("int16", DataType::INT16);
schema->AddDebugField("float", DataType::FLOAT);
schema->AddDebugField("json", DataType::JSON);
schema->AddDebugField("geometry", DataType::GEOMETRY);
schema->AddDebugField("array", DataType::ARRAY, DataType::INT64);
schema->set_primary_field_id(counter_id);
auto int8_nullable_id =
@ -584,6 +585,7 @@ TEST(Sealed, ClearData) {
schema->AddDebugField("int16", DataType::INT16);
schema->AddDebugField("float", DataType::FLOAT);
schema->AddDebugField("json", DataType::JSON);
schema->AddDebugField("geometry", DataType::GEOMETRY);
schema->AddDebugField("array", DataType::ARRAY, DataType::INT64);
schema->set_primary_field_id(counter_id);
@ -715,6 +717,7 @@ TEST(Sealed, LoadFieldDataMmap) {
schema->AddDebugField("int16", DataType::INT16);
schema->AddDebugField("float", DataType::FLOAT);
schema->AddDebugField("json", DataType::JSON);
schema->AddDebugField("geometry", DataType::GEOMETRY);
schema->AddDebugField("array", DataType::ARRAY, DataType::INT64);
schema->set_primary_field_id(counter_id);
@ -2179,6 +2182,7 @@ TEST(Sealed, QueryAllFields) {
auto double_field = schema->AddDebugField("double", DataType::DOUBLE);
auto varchar_field = schema->AddDebugField("varchar", DataType::VARCHAR);
auto json_field = schema->AddDebugField("json", DataType::JSON);
auto geometry_field = schema->AddDebugField("geometry", DataType::GEOMETRY);
auto int_array_field =
schema->AddDebugField("int_array", DataType::ARRAY, DataType::INT8);
auto long_array_field =
@ -2226,6 +2230,7 @@ TEST(Sealed, QueryAllFields) {
auto double_values = dataset.get_col<double>(double_field);
auto varchar_values = dataset.get_col<std::string>(varchar_field);
auto json_values = dataset.get_col<std::string>(json_field);
auto geometry_values = dataset.get_col<std::string>(geometry_field);
auto int_array_values = dataset.get_col<ScalarArray>(int_array_field);
auto long_array_values = dataset.get_col<ScalarArray>(long_array_field);
auto bool_array_values = dataset.get_col<ScalarArray>(bool_array_field);
@ -2255,6 +2260,8 @@ TEST(Sealed, QueryAllFields) {
segment->bulk_subscript(varchar_field, ids_ds->GetIds(), dataset_size);
auto json_result =
segment->bulk_subscript(json_field, ids_ds->GetIds(), dataset_size);
auto geometry_result =
segment->bulk_subscript(geometry_field, ids_ds->GetIds(), dataset_size);
auto int_array_result = segment->bulk_subscript(
int_array_field, ids_ds->GetIds(), dataset_size);
auto long_array_result = segment->bulk_subscript(
@ -2284,6 +2291,8 @@ TEST(Sealed, QueryAllFields) {
EXPECT_EQ(varchar_result->scalars().string_data().data_size(),
dataset_size);
EXPECT_EQ(json_result->scalars().json_data().data_size(), dataset_size);
EXPECT_EQ(geometry_result->scalars().geometry_data().data_size(),
dataset_size);
EXPECT_EQ(vec_result->vectors().float_vector().data_size(),
dataset_size * dim);
EXPECT_EQ(float16_vec_result->vectors().float16_vector().size(),

130
internal/core/unittest/test_utils/DataGen.h
View File

@ -19,6 +19,7 @@
#include <cmath>
#include <google/protobuf/text_format.h>
#include <gtest/gtest.h>
#include <ogr_geometry.h>
#include "Constants.h"
#include "common/EasyAssert.h"
@ -206,6 +207,14 @@ struct GeneratedData {
std::copy(src_data.begin(), src_data.end(), ret_data);
break;
}
case DataType::GEOMETRY: {
auto ret_data =
reinterpret_cast<std::string*>(ret.data());
auto src_data =
target_field_data.scalars().geometry_data().data();
std::copy(src_data.begin(), src_data.end(), ret_data);
break;
}
default: {
PanicInfo(Unsupported, "unsupported");
}
@ -317,14 +326,95 @@ GenerateRandomSparseFloatVector(size_t rows,
return tensor;
}
inline GeneratedData DataGen(SchemaPtr schema,
int64_t N,
uint64_t seed = 42,
uint64_t ts_offset = 0,
int repeat_count = 1,
int array_len = 10,
bool random_pk = false,
bool random_val = true) {
inline void generateRandomPoint(OGRPoint& point) {
point.setX(static_cast<double>(rand()) / RAND_MAX * 360.0 - 180.0);
point.setY(static_cast<double>(rand()) / RAND_MAX * 180.0 - 90.0);
}
inline void
generateRandomLineString(OGRLineString& lineString, int numPoints) {
for (int i = 0; i < numPoints; ++i) {
OGRPoint point;
generateRandomPoint(point);
lineString.addPoint(&point);
}
}
inline void
generateRandomPolygon(OGRPolygon& polygon, int numperRing) {
OGRLinearRing ring;
generateRandomLineString(ring, numperRing);
ring.closeRings();
polygon.addRing(&ring);
}
inline OGRGeometry*
GenRandomGeometry() {
OGRGeometry* geometry = nullptr;
int geomType = rand() % 6; // Randomly select a geometry type (0 to 6)
switch (geomType) {
case 0: {
OGRPoint point;
generateRandomPoint(point);
geometry = point.clone();
break;
}
case 1: {
OGRLineString lineString;
generateRandomLineString(lineString, 5);
geometry = lineString.clone();
break;
}
case 2: {
OGRPolygon polygon;
generateRandomPolygon(polygon, 5);
geometry = polygon.clone();
break;
}
case 3: {
OGRMultiPoint multiPoint;
for (int i = 0; i < 3; ++i) {
OGRPoint point;
generateRandomPoint(point);
multiPoint.addGeometry(&point);
}
geometry = multiPoint.clone();
break;
}
case 4: {
OGRMultiLineString multiLineString;
for (int i = 0; i < 3; ++i) {
OGRLineString lineString;
generateRandomLineString(lineString, 5);
multiLineString.addGeometry(&lineString);
}
geometry = multiLineString.clone();
break;
}
case 5: {
OGRMultiPolygon multiPolygon;
for (int i = 0; i < 3; ++i) {
OGRPolygon polygon;
generateRandomPolygon(polygon, 5);
multiPolygon.addGeometry(&polygon);
}
geometry = multiPolygon.clone();
break;
}
}
return geometry;
}
inline GeneratedData
DataGen(SchemaPtr schema,
int64_t N,
uint64_t seed = 42,
uint64_t pk_offset = 0,
uint64_t ts_offset = 0,
int repeat_count = 1,
int array_len = 10,
bool random_pk = false,
bool random_val = true) {
using std::vector;
std::default_random_engine random(seed);
std::normal_distribution<> distr(0, 1);
@ -515,6 +605,20 @@ inline GeneratedData DataGen(SchemaPtr schema,
insert_cols(data, N, field_meta);
break;
}
case DataType::GEOMETRY: {
vector<std::string> data(N);
for (int i = 0; i < N / repeat_count; i++) {
OGRGeometry* geo = GenRandomGeometry();
size_t size = geo->WkbSize();
unsigned char* wkb_data = new unsigned char[size];
geo->exportToWkb(wkbNDR, wkb_data);
data[i] = std::string(
reinterpret_cast<const char*>(wkb_data), size);
delete[] wkb_data;
}
insert_cols(data, N, field_meta);
break;
}
case DataType::ARRAY: {
vector<ScalarArray> data(N);
switch (field_meta.get_element_type()) {
@ -1156,6 +1260,16 @@ CreateFieldDataFromDataArray(ssize_t raw_count,
}
break;
}
case DataType::GEOMETRY: {
auto src_data = data->scalars().geometry_data().data();
std::vector<std::string> data_raw(src_data.size());
for (int i = 0; i < src_data.size(); i++) {
auto str = src_data.Get(i);
data_raw[i] = std::move(std::string(str));
}
createFieldData(data_raw.data(), DataType::GEOMETRY, dim);
break;
}
case DataType::ARRAY: {
auto src_data = data->scalars().array_data().data();
std::vector<Array> data_raw(src_data.size());

41
internal/distributed/proxy/httpserver/utils.go
View File

@ -3,6 +3,7 @@ package httpserver
import (
"bytes"
"context"
"encoding/base64"
"encoding/binary"
"fmt"
"math"
@ -14,6 +15,8 @@ import (
"github.com/gin-gonic/gin"
"github.com/spf13/cast"
"github.com/tidwall/gjson"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkt"
"go.uber.org/zap"
"google.golang.org/protobuf/proto"
@ -492,6 +495,26 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
}
case schemapb.DataType_JSON:
reallyData[fieldName] = []byte(dataString)
case schemapb.DataType_Geometry:
// treat as string(wkt) data,the string data must be valid
WktString, err := base64.StdEncoding.DecodeString(dataString)
if err != nil {
log.Warn("proxy can not decode datastring with base64", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
fmt.Println("before unmarshal wkt:", string(WktString))
geomT, err := wkt.Unmarshal(string(WktString))
if err != nil {
log.Warn("proxy change wkt to geomtyr failed!!", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
// translate the wkt bytes to wkb bytes ,store the bytes in LittleEndian which cpp core used as well
dataWkbBytes, err := wkb.Marshal(geomT, wkb.NDR)
if err != nil {
log.Warn("proxy change geomtry to wkb failed!!", zap.String("WktString:", dataString))
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray, validDataMap
}
reallyData[fieldName] = dataWkbBytes
case schemapb.DataType_Float:
result, err := cast.ToFloat32E(dataString)
if err != nil {
@ -694,6 +717,8 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
data = make([]*schemapb.ScalarField, 0, rowsLen)
case schemapb.DataType_JSON:
data = make([][]byte, 0, rowsLen)
case schemapb.DataType_Geometry:
data = make([][]byte, 0, rowsLen)
case schemapb.DataType_FloatVector:
data = make([][]float32, 0, rowsLen)
dim, _ := getDim(field)
@ -780,6 +805,8 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
nameColumns[field.Name] = append(nameColumns[field.Name].([]*schemapb.ScalarField), candi.v.Interface().(*schemapb.ScalarField))
case schemapb.DataType_JSON:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
case schemapb.DataType_Geometry:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
case schemapb.DataType_FloatVector:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]float32), candi.v.Interface().([]float32))
case schemapb.DataType_BinaryVector:
@ -928,6 +955,16 @@ func anyToColumns(rows []map[string]interface{}, validDataMap map[string][]bool,
},
},
}
case schemapb.DataType_Geometry:
colData.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: column.([][]byte),
},
},
},
}
case schemapb.DataType_FloatVector:
dim := nameDims[name]
arr, err := convertFloatVectorToArray(column.([][]float32), dim)
@ -1188,6 +1225,8 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
rowsNum = int64(len(fieldDataList[0].GetScalars().GetArrayData().Data))
case schemapb.DataType_JSON:
rowsNum = int64(len(fieldDataList[0].GetScalars().GetJsonData().Data))
case schemapb.DataType_Geometry:
rowsNum = int64(len(fieldDataList[0].GetScalars().GetGeometryData().Data))
case schemapb.DataType_BinaryVector:
rowsNum = int64(len(fieldDataList[0].GetVectors().GetBinaryVector())*8) / fieldDataList[0].GetVectors().GetDim()
case schemapb.DataType_FloatVector:
@ -1326,6 +1365,8 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
}
}
}
case schemapb.DataType_Geometry:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetGeometryData().Data[i]
default:
row[fieldDataList[j].FieldName] = ""
}

36
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -781,7 +781,7 @@ func compareRow(m1 map[string]interface{}, m2 map[string]interface{}) bool {
return false
}
}
} else if key == "field-json" {
} else if key == "field-json" || key == "field-geometry" {
arr1 := value.([]byte)
arr2 := m2[key].([]byte)
if len(arr1) != len(arr2) {
@ -800,7 +800,7 @@ func compareRow(m1 map[string]interface{}, m2 map[string]interface{}) bool {
}
for key, value := range m2 {
if (key == FieldBookIntro) || (key == "field-json") || (key == "field-array") {
if (key == FieldBookIntro) || (key == "field-json") || (key == "field-geometry") || (key == "field-array") {
continue
} else if strings.HasPrefix(key, "array-") {
continue
@ -900,6 +900,12 @@ func newCollectionSchema(coll *schemapb.CollectionSchema) *schemapb.CollectionSc
}
coll.Fields = append(coll.Fields, &fieldSchema10)
fieldSchema11 := schemapb.FieldSchema{
Name: "field-geometry",
DataType: schemapb.DataType_Geometry,
IsDynamic: false,
}
coll.Fields = append(coll.Fields, &fieldSchema11)
return coll
}
@ -1146,6 +1152,28 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
}
fieldDatas = append(fieldDatas, &fieldData11)
fieldData12 := schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldName: "field-geometry",
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
// WKT: POINT (30.123 -10.456)
Data: [][]byte{
[]byte(`POINT (30.123 -10.456)`),
[]byte(`POINT (30.123 -10.456)`),
[]byte(`POINT (30.123 -10.456)`),
// wkb:{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x4},
},
},
},
},
},
IsDynamic: false,
}
fieldDatas = append(fieldDatas, &fieldData12)
switch firstFieldType {
case schemapb.DataType_None:
return fieldDatas
@ -1181,6 +1209,8 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
return []*schemapb.FieldData{&fieldData10}
case schemapb.DataType_JSON:
return []*schemapb.FieldData{&fieldData9}
case schemapb.DataType_Geometry:
return []*schemapb.FieldData{&fieldData12}
case schemapb.DataType_SparseFloatVector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
@ -1446,6 +1476,7 @@ func newSearchResult(results []map[string]interface{}) []map[string]interface{}
result["field-varchar"] = strconv.Itoa(i)
result["field-string"] = strconv.Itoa(i)
result["field-json"] = []byte(`{"XXX": 0}`)
result["field-geometry"] = []byte(`POINT (30.123 -10.456)`)
result["field-array"] = []bool{true}
result["array-bool"] = []bool{true}
result["array-int8"] = []int32{0}
@ -1706,6 +1737,7 @@ func TestBuildQueryResps(t *testing.T) {
schemapb.DataType_Float, schemapb.DataType_Double,
schemapb.DataType_String, schemapb.DataType_VarChar,
schemapb.DataType_JSON, schemapb.DataType_Array,
schemapb.DataType_Geometry,
}
for _, dateType := range dataTypes {
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, dateType), generateIDs(schemapb.DataType_Int64, 3), DefaultScores, true)

15
internal/parser/planparserv2/Plan.g4
View File

@ -23,6 +23,13 @@ expr:
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
| STEuqals'('Identifier','StringLiteral')' # STEuqals
| STTouches'('Identifier','StringLiteral')' # STTouches
| STOverlaps'('Identifier','StringLiteral')' # STOverlaps
| STCrosses'('Identifier','StringLiteral')' # STCrosses
| STContains'('Identifier','StringLiteral')' # STContains
| STIntersects'('Identifier','StringLiteral')' # STIntersects
| STWithin'('Identifier','StringLiteral')' # STWithin
| ArrayLength'('(Identifier | JSONIdentifier)')' # ArrayLength
| Identifier '(' ( expr (',' expr )* ','? )? ')' # Call
| expr op1 = (LT | LE) (Identifier | JSONIdentifier) op2 = (LT | LE) expr # Range
@ -89,6 +96,14 @@ ArrayContainsAll: 'array_contains_all' | 'ARRAY_CONTAINS_ALL';
ArrayContainsAny: 'array_contains_any' | 'ARRAY_CONTAINS_ANY';
ArrayLength: 'array_length' | 'ARRAY_LENGTH';
STEuqals:'st_equals' | 'ST_EQUALS';
STTouches:'st_touches' | 'ST_TOUCHES';
STOverlaps: 'st_overlaps' | 'ST_OVERLAPS';
STCrosses: 'st_crosses' | 'ST_CROSSES';
STContains: 'st_contains' | 'ST_CONTAINS';
STIntersects : 'st_intersects' | 'ST_INTERSECTS';
STWithin :'st_within' | 'ST_WITHIN';
BooleanConstant: 'true' | 'True' | 'TRUE' | 'false' | 'False' | 'FALSE';
IntegerConstant:

16
internal/parser/planparserv2/generated/Plan.interp
View File

File diff suppressed because one or more lines are too long

27
internal/parser/planparserv2/generated/Plan.tokens
View File

@ -38,15 +38,22 @@ ArrayContains=37
ArrayContainsAll=38
ArrayContainsAny=39
ArrayLength=40
BooleanConstant=41
IntegerConstant=42
FloatingConstant=43
Identifier=44
Meta=45
StringLiteral=46
JSONIdentifier=47
Whitespace=48
Newline=49
STEuqals=41
STTouches=42
STOverlaps=43
STCrosses=44
STContains=45
STIntersects=46
STWithin=47
BooleanConstant=48
IntegerConstant=49
FloatingConstant=50
Identifier=51
Meta=52
StringLiteral=53
JSONIdentifier=54
Whitespace=55
Newline=56
'('=1
')'=2
'['=3
@ -72,4 +79,4 @@ Newline=49
'|'=26
'^'=27
'~'=30
'$meta'=45
'$meta'=52

23
internal/parser/planparserv2/generated/PlanLexer.interp
View File

File diff suppressed because one or more lines are too long

27
internal/parser/planparserv2/generated/PlanLexer.tokens
View File

@ -38,15 +38,22 @@ ArrayContains=37
ArrayContainsAll=38
ArrayContainsAny=39
ArrayLength=40
BooleanConstant=41
IntegerConstant=42
FloatingConstant=43
Identifier=44
Meta=45
StringLiteral=46
JSONIdentifier=47
Whitespace=48
Newline=49
STEuqals=41
STTouches=42
STOverlaps=43
STCrosses=44
STContains=45
STIntersects=46
STWithin=47
BooleanConstant=48
IntegerConstant=49
FloatingConstant=50
Identifier=51
Meta=52
StringLiteral=53
JSONIdentifier=54
Whitespace=55
Newline=56
'('=1
')'=2
'['=3
@ -72,4 +79,4 @@ Newline=49
'|'=26
'^'=27
'~'=30
'$meta'=45
'$meta'=52

28
internal/parser/planparserv2/generated/plan_base_visitor.go
View File

@ -39,6 +39,10 @@ func (v *BasePlanVisitor) VisitIdentifier(ctx *IdentifierContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTIntersects(ctx *STIntersectsContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitLike(ctx *LikeContext) interface{} {
return v.VisitChildren(ctx)
}
@ -67,6 +71,10 @@ func (v *BasePlanVisitor) VisitCall(ctx *CallContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTCrosses(ctx *STCrossesContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitReverseRange(ctx *ReverseRangeContext) interface{} {
return v.VisitChildren(ctx)
}
@ -95,6 +103,14 @@ func (v *BasePlanVisitor) VisitTextMatch(ctx *TextMatchContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTTouches(ctx *STTouchesContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTContains(ctx *STContainsContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitTerm(ctx *TermContext) interface{} {
return v.VisitChildren(ctx)
}
@ -103,6 +119,10 @@ func (v *BasePlanVisitor) VisitJSONContains(ctx *JSONContainsContext) interface{
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTWithin(ctx *STWithinContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitRange(ctx *RangeContext) interface{} {
return v.VisitChildren(ctx)
}
@ -135,6 +155,14 @@ func (v *BasePlanVisitor) VisitBitAnd(ctx *BitAndContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTEuqals(ctx *STEuqalsContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitPower(ctx *PowerContext) interface{} {
return v.VisitChildren(ctx)
}
func (v *BasePlanVisitor) VisitSTOverlaps(ctx *STOverlapsContext) interface{} {
return v.VisitChildren(ctx)
}

747
internal/parser/planparserv2/generated/plan_lexer.go
View File

@ -46,7 +46,8 @@ func planlexerLexerInit() {
"", "'('", "')'", "'['", "','", "']'", "'{'", "'}'", "'<'", "'<='",
"'>'", "'>='", "'=='", "'!='", "", "", "", "'+'", "'-'", "'*'", "'/'",
"'%'", "'**'", "'<<'", "'>>'", "'&'", "'|'", "'^'", "", "", "'~'", "",
"", "", "", "", "", "", "", "", "", "", "", "", "", "'$meta'",
"", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "",
"", "", "", "'$meta'",
}
staticData.SymbolicNames = []string{
"", "", "", "", "", "", "LBRACE", "RBRACE", "LT", "LE", "GT", "GE",
@ -54,8 +55,10 @@ func planlexerLexerInit() {
"MOD", "POW", "SHL", "SHR", "BAND", "BOR", "BXOR", "AND", "OR", "BNOT",
"NOT", "IN", "EmptyArray", "JSONContains", "JSONContainsAll", "JSONContainsAny",
"ArrayContains", "ArrayContainsAll", "ArrayContainsAny", "ArrayLength",
"BooleanConstant", "IntegerConstant", "FloatingConstant", "Identifier",
"Meta", "StringLiteral", "JSONIdentifier", "Whitespace", "Newline",
"STEuqals", "STTouches", "STOverlaps", "STCrosses", "STContains", "STIntersects",
"STWithin", "BooleanConstant", "IntegerConstant", "FloatingConstant",
"Identifier", "Meta", "StringLiteral", "JSONIdentifier", "Whitespace",
"Newline",
}
staticData.RuleNames = []string{
"T__0", "T__1", "T__2", "T__3", "T__4", "LBRACE", "RBRACE", "LT", "LE",
@ -63,7 +66,8 @@ func planlexerLexerInit() {
"MUL", "DIV", "MOD", "POW", "SHL", "SHR", "BAND", "BOR", "BXOR", "AND",
"OR", "BNOT", "NOT", "IN", "EmptyArray", "JSONContains", "JSONContainsAll",
"JSONContainsAny", "ArrayContains", "ArrayContainsAll", "ArrayContainsAny",
"ArrayLength", "BooleanConstant", "IntegerConstant", "FloatingConstant",
"ArrayLength", "STEuqals", "STTouches", "STOverlaps", "STCrosses", "STContains",
"STIntersects", "STWithin", "BooleanConstant", "IntegerConstant", "FloatingConstant",
"Identifier", "Meta", "StringLiteral", "JSONIdentifier", "EncodingPrefix",
"DoubleSCharSequence", "SingleSCharSequence", "DoubleSChar", "SingleSChar",
"Nondigit", "Digit", "BinaryConstant", "DecimalConstant", "OctalConstant",
@ -75,7 +79,7 @@ func planlexerLexerInit() {
}
staticData.PredictionContextCache = antlr.NewPredictionContextCache()
staticData.serializedATN = []int32{
4, 0, 49, 789, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
4, 0, 56, 963, 6, -1, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2,
4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2,
10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15,
7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7,
@ -89,87 +93,105 @@ func planlexerLexerInit() {
7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 2, 61, 7, 61, 2, 62, 7,
62, 2, 63, 7, 63, 2, 64, 7, 64, 2, 65, 7, 65, 2, 66, 7, 66, 2, 67, 7, 67,
2, 68, 7, 68, 2, 69, 7, 69, 2, 70, 7, 70, 2, 71, 7, 71, 2, 72, 7, 72, 2,
73, 7, 73, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2, 1, 3, 1, 3, 1, 4, 1, 4,
1, 5, 1, 5, 1, 6, 1, 6, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 10,
1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1,
13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 188, 8, 13, 1, 14, 1, 14,
1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 3,
14, 202, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15,
73, 7, 73, 2, 74, 7, 74, 2, 75, 7, 75, 2, 76, 7, 76, 2, 77, 7, 77, 2, 78,
7, 78, 2, 79, 7, 79, 2, 80, 7, 80, 1, 0, 1, 0, 1, 1, 1, 1, 1, 2, 1, 2,
1, 3, 1, 3, 1, 4, 1, 4, 1, 5, 1, 5, 1, 6, 1, 6, 1, 7, 1, 7, 1, 8, 1, 8,
1, 8, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 12, 1, 12,
1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 1, 13, 3, 13, 202,
8, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1, 14, 1,
14, 1, 14, 1, 14, 3, 14, 216, 8, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15,
1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1,
15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 231,
8, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1, 19, 1, 19, 1, 20, 1,
20, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 23, 1, 23, 1, 23, 1, 24,
1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 27, 3,
27, 263, 8, 27, 1, 28, 1, 28, 1, 28, 1, 28, 3, 28, 269, 8, 28, 1, 29, 1,
29, 1, 30, 1, 30, 1, 30, 1, 30, 3, 30, 277, 8, 30, 1, 31, 1, 31, 1, 31,
1, 31, 3, 31, 283, 8, 31, 1, 32, 1, 32, 1, 32, 5, 32, 288, 8, 32, 10, 32,
12, 32, 291, 9, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1,
15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15,
1, 15, 3, 15, 245, 8, 15, 1, 16, 1, 16, 1, 17, 1, 17, 1, 18, 1, 18, 1,
19, 1, 19, 1, 20, 1, 20, 1, 21, 1, 21, 1, 21, 1, 22, 1, 22, 1, 22, 1, 23,
1, 23, 1, 23, 1, 24, 1, 24, 1, 25, 1, 25, 1, 26, 1, 26, 1, 27, 1, 27, 1,
27, 1, 27, 1, 27, 3, 27, 277, 8, 27, 1, 28, 1, 28, 1, 28, 1, 28, 3, 28,
283, 8, 28, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 3, 30, 291, 8, 30,
1, 31, 1, 31, 1, 31, 1, 31, 3, 31, 297, 8, 31, 1, 32, 1, 32, 1, 32, 5,
32, 302, 8, 32, 10, 32, 12, 32, 305, 9, 32, 1, 32, 1, 32, 1, 33, 1, 33,
1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1,
33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33,
1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 3,
33, 321, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34,
1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1,
1, 33, 1, 33, 1, 33, 3, 33, 335, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1,
34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34,
1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 357, 8, 34, 1, 35, 1, 35, 1,
1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1,
34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 371,
8, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1,
35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35,
1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1,
35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35,
3, 35, 393, 8, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1,
35, 1, 35, 1, 35, 1, 35, 3, 35, 407, 8, 35, 1, 36, 1, 36, 1, 36, 1, 36,
1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1,
36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36,
1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 3,
36, 423, 8, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37,
1, 36, 1, 36, 1, 36, 3, 36, 437, 8, 36, 1, 37, 1, 37, 1, 37, 1, 37, 1,
37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37,
1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1,
37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37,
1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 3, 37, 461, 8, 37, 1,
3, 37, 475, 8, 37, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1,
38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38,
1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1,
38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38,
1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 499, 8, 38, 1, 39, 1, 39, 1, 39, 1,
38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 513, 8, 38,
1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1,
39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39,
1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3,
39, 525, 8, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1,
40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 3, 40, 554,
8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 560, 8, 41, 1, 42, 1, 42, 3,
42, 564, 8, 42, 1, 43, 1, 43, 1, 43, 5, 43, 569, 8, 43, 10, 43, 12, 43,
572, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 45, 3, 45, 581,
8, 45, 1, 45, 1, 45, 3, 45, 585, 8, 45, 1, 45, 1, 45, 1, 45, 3, 45, 590,
8, 45, 1, 45, 3, 45, 593, 8, 45, 1, 46, 1, 46, 3, 46, 597, 8, 46, 1, 46,
1, 46, 1, 46, 3, 46, 602, 8, 46, 1, 46, 1, 46, 4, 46, 606, 8, 46, 11, 46,
12, 46, 607, 1, 47, 1, 47, 1, 47, 3, 47, 613, 8, 47, 1, 48, 4, 48, 616,
8, 48, 11, 48, 12, 48, 617, 1, 49, 4, 49, 621, 8, 49, 11, 49, 12, 49, 622,
1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 3, 50, 632, 8, 50, 1,
51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 3, 51, 641, 8, 51, 1, 52,
1, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 54, 4, 54, 650, 8, 54, 11, 54, 12,
54, 651, 1, 55, 1, 55, 5, 55, 656, 8, 55, 10, 55, 12, 55, 659, 9, 55, 1,
55, 3, 55, 662, 8, 55, 1, 56, 1, 56, 5, 56, 666, 8, 56, 10, 56, 12, 56,
669, 9, 56, 1, 57, 1, 57, 1, 57, 1, 57, 1, 58, 1, 58, 1, 59, 1, 59, 1,
60, 1, 60, 1, 61, 1, 61, 1, 61, 1, 61, 1, 61, 1, 62, 1, 62, 1, 62, 1, 62,
1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 1, 62, 3, 62, 696, 8, 62, 1, 63, 1,
63, 3, 63, 700, 8, 63, 1, 63, 1, 63, 1, 63, 3, 63, 705, 8, 63, 1, 64, 1,
64, 1, 64, 1, 64, 3, 64, 711, 8, 64, 1, 64, 1, 64, 1, 65, 3, 65, 716, 8,
65, 1, 65, 1, 65, 1, 65, 1, 65, 1, 65, 3, 65, 723, 8, 65, 1, 66, 1, 66,
3, 66, 727, 8, 66, 1, 66, 1, 66, 1, 67, 4, 67, 732, 8, 67, 11, 67, 12,
67, 733, 1, 68, 3, 68, 737, 8, 68, 1, 68, 1, 68, 1, 68, 1, 68, 1, 68, 3,
68, 744, 8, 68, 1, 69, 4, 69, 747, 8, 69, 11, 69, 12, 69, 748, 1, 70, 1,
70, 3, 70, 753, 8, 70, 1, 70, 1, 70, 1, 71, 1, 71, 1, 71, 1, 71, 1, 71,
3, 71, 762, 8, 71, 1, 71, 3, 71, 765, 8, 71, 1, 71, 1, 71, 1, 71, 1, 71,
1, 71, 3, 71, 772, 8, 71, 1, 72, 4, 72, 775, 8, 72, 11, 72, 12, 72, 776,
1, 72, 1, 72, 1, 73, 1, 73, 3, 73, 783, 8, 73, 1, 73, 3, 73, 786, 8, 73,
1, 73, 1, 73, 0, 0, 74, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13, 7, 15,
8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 14, 29, 15, 31, 16, 33, 17,
35, 18, 37, 19, 39, 20, 41, 21, 43, 22, 45, 23, 47, 24, 49, 25, 51, 26,
53, 27, 55, 28, 57, 29, 59, 30, 61, 31, 63, 32, 65, 33, 67, 34, 69, 35,
71, 36, 73, 37, 75, 38, 77, 39, 79, 40, 81, 41, 83, 42, 85, 43, 87, 44,
89, 45, 91, 46, 93, 47, 95, 0, 97, 0, 99, 0, 101, 0, 103, 0, 105, 0, 107,
0, 109, 0, 111, 0, 113, 0, 115, 0, 117, 0, 119, 0, 121, 0, 123, 0, 125,
0, 127, 0, 129, 0, 131, 0, 133, 0, 135, 0, 137, 0, 139, 0, 141, 0, 143,
0, 145, 48, 147, 49, 1, 0, 16, 3, 0, 76, 76, 85, 85, 117, 117, 4, 0, 10,
1, 39, 1, 39, 1, 39, 3, 39, 539, 8, 39, 1, 40, 1, 40, 1, 40, 1, 40, 1,
40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40, 1, 40,
1, 40, 1, 40, 1, 40, 3, 40, 559, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 1,
41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41,
1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 3, 41, 581, 8, 41, 1, 42, 1, 42, 1,
42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42,
1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 605,
8, 42, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1,
43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43, 1, 43,
3, 43, 627, 8, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1,
44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44,
1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 651, 8, 44, 1, 45, 1, 45, 1, 45, 1,
45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45,
1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1,
45, 1, 45, 3, 45, 679, 8, 45, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46,
1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1, 46, 1,
46, 1, 46, 3, 46, 699, 8, 46, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47,
1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1,
47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47, 1, 47,
3, 47, 728, 8, 47, 1, 48, 1, 48, 1, 48, 1, 48, 3, 48, 734, 8, 48, 1, 49,
1, 49, 3, 49, 738, 8, 49, 1, 50, 1, 50, 1, 50, 5, 50, 743, 8, 50, 10, 50,
12, 50, 746, 9, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 52, 3,
52, 755, 8, 52, 1, 52, 1, 52, 3, 52, 759, 8, 52, 1, 52, 1, 52, 1, 52, 3,
52, 764, 8, 52, 1, 52, 3, 52, 767, 8, 52, 1, 53, 1, 53, 3, 53, 771, 8,
53, 1, 53, 1, 53, 1, 53, 3, 53, 776, 8, 53, 1, 53, 1, 53, 4, 53, 780, 8,
53, 11, 53, 12, 53, 781, 1, 54, 1, 54, 1, 54, 3, 54, 787, 8, 54, 1, 55,
4, 55, 790, 8, 55, 11, 55, 12, 55, 791, 1, 56, 4, 56, 795, 8, 56, 11, 56,
12, 56, 796, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 1, 57, 3, 57, 806,
8, 57, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 1, 58, 3, 58, 815, 8,
58, 1, 59, 1, 59, 1, 60, 1, 60, 1, 61, 1, 61, 1, 61, 4, 61, 824, 8, 61,
11, 61, 12, 61, 825, 1, 62, 1, 62, 5, 62, 830, 8, 62, 10, 62, 12, 62, 833,
9, 62, 1, 62, 3, 62, 836, 8, 62, 1, 63, 1, 63, 5, 63, 840, 8, 63, 10, 63,
12, 63, 843, 9, 63, 1, 64, 1, 64, 1, 64, 1, 64, 1, 65, 1, 65, 1, 66, 1,
66, 1, 67, 1, 67, 1, 68, 1, 68, 1, 68, 1, 68, 1, 68, 1, 69, 1, 69, 1, 69,
1, 69, 1, 69, 1, 69, 1, 69, 1, 69, 1, 69, 1, 69, 3, 69, 870, 8, 69, 1,
70, 1, 70, 3, 70, 874, 8, 70, 1, 70, 1, 70, 1, 70, 3, 70, 879, 8, 70, 1,
71, 1, 71, 1, 71, 1, 71, 3, 71, 885, 8, 71, 1, 71, 1, 71, 1, 72, 3, 72,
890, 8, 72, 1, 72, 1, 72, 1, 72, 1, 72, 1, 72, 3, 72, 897, 8, 72, 1, 73,
1, 73, 3, 73, 901, 8, 73, 1, 73, 1, 73, 1, 74, 4, 74, 906, 8, 74, 11, 74,
12, 74, 907, 1, 75, 3, 75, 911, 8, 75, 1, 75, 1, 75, 1, 75, 1, 75, 1, 75,
3, 75, 918, 8, 75, 1, 76, 4, 76, 921, 8, 76, 11, 76, 12, 76, 922, 1, 77,
1, 77, 3, 77, 927, 8, 77, 1, 77, 1, 77, 1, 78, 1, 78, 1, 78, 1, 78, 1,
78, 3, 78, 936, 8, 78, 1, 78, 3, 78, 939, 8, 78, 1, 78, 1, 78, 1, 78, 1,
78, 1, 78, 3, 78, 946, 8, 78, 1, 79, 4, 79, 949, 8, 79, 11, 79, 12, 79,
950, 1, 79, 1, 79, 1, 80, 1, 80, 3, 80, 957, 8, 80, 1, 80, 3, 80, 960,
8, 80, 1, 80, 1, 80, 0, 0, 81, 1, 1, 3, 2, 5, 3, 7, 4, 9, 5, 11, 6, 13,
7, 15, 8, 17, 9, 19, 10, 21, 11, 23, 12, 25, 13, 27, 14, 29, 15, 31, 16,
33, 17, 35, 18, 37, 19, 39, 20, 41, 21, 43, 22, 45, 23, 47, 24, 49, 25,
51, 26, 53, 27, 55, 28, 57, 29, 59, 30, 61, 31, 63, 32, 65, 33, 67, 34,
69, 35, 71, 36, 73, 37, 75, 38, 77, 39, 79, 40, 81, 41, 83, 42, 85, 43,
87, 44, 89, 45, 91, 46, 93, 47, 95, 48, 97, 49, 99, 50, 101, 51, 103, 52,
105, 53, 107, 54, 109, 0, 111, 0, 113, 0, 115, 0, 117, 0, 119, 0, 121,
0, 123, 0, 125, 0, 127, 0, 129, 0, 131, 0, 133, 0, 135, 0, 137, 0, 139,
0, 141, 0, 143, 0, 145, 0, 147, 0, 149, 0, 151, 0, 153, 0, 155, 0, 157,
0, 159, 55, 161, 56, 1, 0, 16, 3, 0, 76, 76, 85, 85, 117, 117, 4, 0, 10,
10, 13, 13, 34, 34, 92, 92, 4, 0, 10, 10, 13, 13, 39, 39, 92, 92, 3, 0,
65, 90, 95, 95, 97, 122, 1, 0, 48, 57, 2, 0, 66, 66, 98, 98, 1, 0, 48,
49, 2, 0, 88, 88, 120, 120, 1, 0, 49, 57, 1, 0, 48, 55, 3, 0, 48, 57, 65,
70, 97, 102, 2, 0, 69, 69, 101, 101, 2, 0, 43, 43, 45, 45, 2, 0, 80, 80,
112, 112, 10, 0, 34, 34, 39, 39, 63, 63, 92, 92, 97, 98, 102, 102, 110,
110, 114, 114, 116, 116, 118, 118, 2, 0, 9, 9, 32, 32, 831, 0, 1, 1, 0,
110, 114, 114, 116, 116, 118, 118, 2, 0, 9, 9, 32, 32, 1012, 0, 1, 1, 0,
0, 0, 0, 3, 1, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 9, 1, 0,
0, 0, 0, 11, 1, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 17, 1,
0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 23, 1, 0, 0, 0, 0, 25,
@ -182,257 +204,315 @@ func planlexerLexerInit() {
1, 0, 0, 0, 0, 73, 1, 0, 0, 0, 0, 75, 1, 0, 0, 0, 0, 77, 1, 0, 0, 0, 0,
79, 1, 0, 0, 0, 0, 81, 1, 0, 0, 0, 0, 83, 1, 0, 0, 0, 0, 85, 1, 0, 0, 0,
0, 87, 1, 0, 0, 0, 0, 89, 1, 0, 0, 0, 0, 91, 1, 0, 0, 0, 0, 93, 1, 0, 0,
0, 0, 145, 1, 0, 0, 0, 0, 147, 1, 0, 0, 0, 1, 149, 1, 0, 0, 0, 3, 151,
1, 0, 0, 0, 5, 153, 1, 0, 0, 0, 7, 155, 1, 0, 0, 0, 9, 157, 1, 0, 0, 0,
11, 159, 1, 0, 0, 0, 13, 161, 1, 0, 0, 0, 15, 163, 1, 0, 0, 0, 17, 165,
1, 0, 0, 0, 19, 168, 1, 0, 0, 0, 21, 170, 1, 0, 0, 0, 23, 173, 1, 0, 0,
0, 25, 176, 1, 0, 0, 0, 27, 187, 1, 0, 0, 0, 29, 201, 1, 0, 0, 0, 31, 230,
1, 0, 0, 0, 33, 232, 1, 0, 0, 0, 35, 234, 1, 0, 0, 0, 37, 236, 1, 0, 0,
0, 39, 238, 1, 0, 0, 0, 41, 240, 1, 0, 0, 0, 43, 242, 1, 0, 0, 0, 45, 245,
1, 0, 0, 0, 47, 248, 1, 0, 0, 0, 49, 251, 1, 0, 0, 0, 51, 253, 1, 0, 0,
0, 53, 255, 1, 0, 0, 0, 55, 262, 1, 0, 0, 0, 57, 268, 1, 0, 0, 0, 59, 270,
1, 0, 0, 0, 61, 276, 1, 0, 0, 0, 63, 282, 1, 0, 0, 0, 65, 284, 1, 0, 0,
0, 67, 320, 1, 0, 0, 0, 69, 356, 1, 0, 0, 0, 71, 392, 1, 0, 0, 0, 73, 422,
1, 0, 0, 0, 75, 460, 1, 0, 0, 0, 77, 498, 1, 0, 0, 0, 79, 524, 1, 0, 0,
0, 81, 553, 1, 0, 0, 0, 83, 559, 1, 0, 0, 0, 85, 563, 1, 0, 0, 0, 87, 565,
1, 0, 0, 0, 89, 573, 1, 0, 0, 0, 91, 580, 1, 0, 0, 0, 93, 596, 1, 0, 0,
0, 95, 612, 1, 0, 0, 0, 97, 615, 1, 0, 0, 0, 99, 620, 1, 0, 0, 0, 101,
631, 1, 0, 0, 0, 103, 640, 1, 0, 0, 0, 105, 642, 1, 0, 0, 0, 107, 644,
1, 0, 0, 0, 109, 646, 1, 0, 0, 0, 111, 661, 1, 0, 0, 0, 113, 663, 1, 0,
0, 0, 115, 670, 1, 0, 0, 0, 117, 674, 1, 0, 0, 0, 119, 676, 1, 0, 0, 0,
121, 678, 1, 0, 0, 0, 123, 680, 1, 0, 0, 0, 125, 695, 1, 0, 0, 0, 127,
704, 1, 0, 0, 0, 129, 706, 1, 0, 0, 0, 131, 722, 1, 0, 0, 0, 133, 724,
1, 0, 0, 0, 135, 731, 1, 0, 0, 0, 137, 743, 1, 0, 0, 0, 139, 746, 1, 0,
0, 0, 141, 750, 1, 0, 0, 0, 143, 771, 1, 0, 0, 0, 145, 774, 1, 0, 0, 0,
147, 785, 1, 0, 0, 0, 149, 150, 5, 40, 0, 0, 150, 2, 1, 0, 0, 0, 151, 152,
5, 41, 0, 0, 152, 4, 1, 0, 0, 0, 153, 154, 5, 91, 0, 0, 154, 6, 1, 0, 0,
0, 155, 156, 5, 44, 0, 0, 156, 8, 1, 0, 0, 0, 157, 158, 5, 93, 0, 0, 158,
10, 1, 0, 0, 0, 159, 160, 5, 123, 0, 0, 160, 12, 1, 0, 0, 0, 161, 162,
5, 125, 0, 0, 162, 14, 1, 0, 0, 0, 163, 164, 5, 60, 0, 0, 164, 16, 1, 0,
0, 0, 165, 166, 5, 60, 0, 0, 166, 167, 5, 61, 0, 0, 167, 18, 1, 0, 0, 0,
168, 169, 5, 62, 0, 0, 169, 20, 1, 0, 0, 0, 170, 171, 5, 62, 0, 0, 171,
172, 5, 61, 0, 0, 172, 22, 1, 0, 0, 0, 173, 174, 5, 61, 0, 0, 174, 175,
5, 61, 0, 0, 175, 24, 1, 0, 0, 0, 176, 177, 5, 33, 0, 0, 177, 178, 5, 61,
0, 0, 178, 26, 1, 0, 0, 0, 179, 180, 5, 108, 0, 0, 180, 181, 5, 105, 0,
0, 181, 182, 5, 107, 0, 0, 182, 188, 5, 101, 0, 0, 183, 184, 5, 76, 0,
0, 184, 185, 5, 73, 0, 0, 185, 186, 5, 75, 0, 0, 186, 188, 5, 69, 0, 0,
187, 179, 1, 0, 0, 0, 187, 183, 1, 0, 0, 0, 188, 28, 1, 0, 0, 0, 189, 190,
5, 101, 0, 0, 190, 191, 5, 120, 0, 0, 191, 192, 5, 105, 0, 0, 192, 193,
5, 115, 0, 0, 193, 194, 5, 116, 0, 0, 194, 202, 5, 115, 0, 0, 195, 196,
5, 69, 0, 0, 196, 197, 5, 88, 0, 0, 197, 198, 5, 73, 0, 0, 198, 199, 5,
83, 0, 0, 199, 200, 5, 84, 0, 0, 200, 202, 5, 83, 0, 0, 201, 189, 1, 0,
0, 0, 201, 195, 1, 0, 0, 0, 202, 30, 1, 0, 0, 0, 203, 204, 5, 84, 0, 0,
204, 205, 5, 101, 0, 0, 205, 206, 5, 120, 0, 0, 206, 207, 5, 116, 0, 0,
207, 208, 5, 77, 0, 0, 208, 209, 5, 97, 0, 0, 209, 210, 5, 116, 0, 0, 210,
211, 5, 99, 0, 0, 211, 231, 5, 104, 0, 0, 212, 213, 5, 116, 0, 0, 213,
214, 5, 101, 0, 0, 214, 215, 5, 120, 0, 0, 215, 216, 5, 116, 0, 0, 216,
217, 5, 109, 0, 0, 217, 218, 5, 97, 0, 0, 218, 219, 5, 116, 0, 0, 219,
220, 5, 99, 0, 0, 220, 231, 5, 104, 0, 0, 221, 222, 5, 84, 0, 0, 222, 223,
5, 69, 0, 0, 223, 224, 5, 88, 0, 0, 224, 225, 5, 84, 0, 0, 225, 226, 5,
77, 0, 0, 226, 227, 5, 65, 0, 0, 227, 228, 5, 84, 0, 0, 228, 229, 5, 67,
0, 0, 229, 231, 5, 72, 0, 0, 230, 203, 1, 0, 0, 0, 230, 212, 1, 0, 0, 0,
230, 221, 1, 0, 0, 0, 231, 32, 1, 0, 0, 0, 232, 233, 5, 43, 0, 0, 233,
34, 1, 0, 0, 0, 234, 235, 5, 45, 0, 0, 235, 36, 1, 0, 0, 0, 236, 237, 5,
42, 0, 0, 237, 38, 1, 0, 0, 0, 238, 239, 5, 47, 0, 0, 239, 40, 1, 0, 0,
0, 240, 241, 5, 37, 0, 0, 241, 42, 1, 0, 0, 0, 242, 243, 5, 42, 0, 0, 243,
244, 5, 42, 0, 0, 244, 44, 1, 0, 0, 0, 245, 246, 5, 60, 0, 0, 246, 247,
5, 60, 0, 0, 247, 46, 1, 0, 0, 0, 248, 249, 5, 62, 0, 0, 249, 250, 5, 62,
0, 0, 250, 48, 1, 0, 0, 0, 251, 252, 5, 38, 0, 0, 252, 50, 1, 0, 0, 0,
253, 254, 5, 124, 0, 0, 254, 52, 1, 0, 0, 0, 255, 256, 5, 94, 0, 0, 256,
54, 1, 0, 0, 0, 257, 258, 5, 38, 0, 0, 258, 263, 5, 38, 0, 0, 259, 260,
5, 97, 0, 0, 260, 261, 5, 110, 0, 0, 261, 263, 5, 100, 0, 0, 262, 257,
1, 0, 0, 0, 262, 259, 1, 0, 0, 0, 263, 56, 1, 0, 0, 0, 264, 265, 5, 124,
0, 0, 265, 269, 5, 124, 0, 0, 266, 267, 5, 111, 0, 0, 267, 269, 5, 114,
0, 0, 268, 264, 1, 0, 0, 0, 268, 266, 1, 0, 0, 0, 269, 58, 1, 0, 0, 0,
270, 271, 5, 126, 0, 0, 271, 60, 1, 0, 0, 0, 272, 277, 5, 33, 0, 0, 273,
274, 5, 110, 0, 0, 274, 275, 5, 111, 0, 0, 275, 277, 5, 116, 0, 0, 276,
272, 1, 0, 0, 0, 276, 273, 1, 0, 0, 0, 277, 62, 1, 0, 0, 0, 278, 279, 5,
105, 0, 0, 279, 283, 5, 110, 0, 0, 280, 281, 5, 73, 0, 0, 281, 283, 5,
78, 0, 0, 282, 278, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 64, 1, 0, 0,
0, 284, 289, 5, 91, 0, 0, 285, 288, 3, 145, 72, 0, 286, 288, 3, 147, 73,
0, 287, 285, 1, 0, 0, 0, 287, 286, 1, 0, 0, 0, 288, 291, 1, 0, 0, 0, 289,
287, 1, 0, 0, 0, 289, 290, 1, 0, 0, 0, 290, 292, 1, 0, 0, 0, 291, 289,
1, 0, 0, 0, 292, 293, 5, 93, 0, 0, 293, 66, 1, 0, 0, 0, 294, 295, 5, 106,
0, 0, 295, 296, 5, 115, 0, 0, 296, 297, 5, 111, 0, 0, 297, 298, 5, 110,
0, 0, 298, 299, 5, 95, 0, 0, 299, 300, 5, 99, 0, 0, 300, 301, 5, 111, 0,
0, 301, 302, 5, 110, 0, 0, 302, 303, 5, 116, 0, 0, 303, 304, 5, 97, 0,
0, 304, 305, 5, 105, 0, 0, 305, 306, 5, 110, 0, 0, 306, 321, 5, 115, 0,
0, 307, 308, 5, 74, 0, 0, 308, 309, 5, 83, 0, 0, 309, 310, 5, 79, 0, 0,
310, 311, 5, 78, 0, 0, 311, 312, 5, 95, 0, 0, 312, 313, 5, 67, 0, 0, 313,
314, 5, 79, 0, 0, 314, 315, 5, 78, 0, 0, 315, 316, 5, 84, 0, 0, 316, 317,
5, 65, 0, 0, 317, 318, 5, 73, 0, 0, 318, 319, 5, 78, 0, 0, 319, 321, 5,
83, 0, 0, 320, 294, 1, 0, 0, 0, 320, 307, 1, 0, 0, 0, 321, 68, 1, 0, 0,
0, 322, 323, 5, 106, 0, 0, 323, 324, 5, 115, 0, 0, 324, 325, 5, 111, 0,
0, 325, 326, 5, 110, 0, 0, 326, 327, 5, 95, 0, 0, 327, 328, 5, 99, 0, 0,
328, 329, 5, 111, 0, 0, 329, 330, 5, 110, 0, 0, 330, 331, 5, 116, 0, 0,
331, 332, 5, 97, 0, 0, 332, 333, 5, 105, 0, 0, 333, 334, 5, 110, 0, 0,
334, 335, 5, 115, 0, 0, 335, 336, 5, 95, 0, 0, 336, 337, 5, 97, 0, 0, 337,
338, 5, 108, 0, 0, 338, 357, 5, 108, 0, 0, 339, 340, 5, 74, 0, 0, 340,
341, 5, 83, 0, 0, 341, 342, 5, 79, 0, 0, 342, 343, 5, 78, 0, 0, 343, 344,
5, 95, 0, 0, 344, 345, 5, 67, 0, 0, 345, 346, 5, 79, 0, 0, 346, 347, 5,
78, 0, 0, 347, 348, 5, 84, 0, 0, 348, 349, 5, 65, 0, 0, 349, 350, 5, 73,
0, 0, 350, 351, 5, 78, 0, 0, 351, 352, 5, 83, 0, 0, 352, 353, 5, 95, 0,
0, 353, 354, 5, 65, 0, 0, 354, 355, 5, 76, 0, 0, 355, 357, 5, 76, 0, 0,
356, 322, 1, 0, 0, 0, 356, 339, 1, 0, 0, 0, 357, 70, 1, 0, 0, 0, 358, 359,
5, 106, 0, 0, 359, 360, 5, 115, 0, 0, 360, 361, 5, 111, 0, 0, 361, 362,
5, 110, 0, 0, 362, 363, 5, 95, 0, 0, 363, 364, 5, 99, 0, 0, 364, 365, 5,
111, 0, 0, 365, 366, 5, 110, 0, 0, 366, 367, 5, 116, 0, 0, 367, 368, 5,
97, 0, 0, 368, 369, 5, 105, 0, 0, 369, 370, 5, 110, 0, 0, 370, 371, 5,
115, 0, 0, 371, 372, 5, 95, 0, 0, 372, 373, 5, 97, 0, 0, 373, 374, 5, 110,
0, 0, 374, 393, 5, 121, 0, 0, 375, 376, 5, 74, 0, 0, 376, 377, 5, 83, 0,
0, 377, 378, 5, 79, 0, 0, 378, 379, 5, 78, 0, 0, 379, 380, 5, 95, 0, 0,
380, 381, 5, 67, 0, 0, 381, 382, 5, 79, 0, 0, 382, 383, 5, 78, 0, 0, 383,
384, 5, 84, 0, 0, 384, 385, 5, 65, 0, 0, 385, 386, 5, 73, 0, 0, 386, 387,
5, 78, 0, 0, 387, 388, 5, 83, 0, 0, 388, 389, 5, 95, 0, 0, 389, 390, 5,
65, 0, 0, 390, 391, 5, 78, 0, 0, 391, 393, 5, 89, 0, 0, 392, 358, 1, 0,
0, 0, 392, 375, 1, 0, 0, 0, 393, 72, 1, 0, 0, 0, 394, 395, 5, 97, 0, 0,
395, 396, 5, 114, 0, 0, 396, 397, 5, 114, 0, 0, 397, 398, 5, 97, 0, 0,
398, 399, 5, 121, 0, 0, 399, 400, 5, 95, 0, 0, 400, 401, 5, 99, 0, 0, 401,
402, 5, 111, 0, 0, 402, 403, 5, 110, 0, 0, 403, 404, 5, 116, 0, 0, 404,
405, 5, 97, 0, 0, 405, 406, 5, 105, 0, 0, 406, 407, 5, 110, 0, 0, 407,
423, 5, 115, 0, 0, 408, 409, 5, 65, 0, 0, 409, 410, 5, 82, 0, 0, 410, 411,
5, 82, 0, 0, 411, 412, 5, 65, 0, 0, 412, 413, 5, 89, 0, 0, 413, 414, 5,
95, 0, 0, 414, 415, 5, 67, 0, 0, 415, 416, 5, 79, 0, 0, 416, 417, 5, 78,
0, 0, 417, 418, 5, 84, 0, 0, 418, 419, 5, 65, 0, 0, 419, 420, 5, 73, 0,
0, 420, 421, 5, 78, 0, 0, 421, 423, 5, 83, 0, 0, 422, 394, 1, 0, 0, 0,
422, 408, 1, 0, 0, 0, 423, 74, 1, 0, 0, 0, 424, 425, 5, 97, 0, 0, 425,
426, 5, 114, 0, 0, 426, 427, 5, 114, 0, 0, 427, 428, 5, 97, 0, 0, 428,
429, 5, 121, 0, 0, 429, 430, 5, 95, 0, 0, 430, 431, 5, 99, 0, 0, 431, 432,
5, 111, 0, 0, 432, 433, 5, 110, 0, 0, 433, 434, 5, 116, 0, 0, 434, 435,
5, 97, 0, 0, 435, 436, 5, 105, 0, 0, 436, 437, 5, 110, 0, 0, 437, 438,
5, 115, 0, 0, 438, 439, 5, 95, 0, 0, 439, 440, 5, 97, 0, 0, 440, 441, 5,
108, 0, 0, 441, 461, 5, 108, 0, 0, 442, 443, 5, 65, 0, 0, 443, 444, 5,
82, 0, 0, 444, 445, 5, 82, 0, 0, 445, 446, 5, 65, 0, 0, 446, 447, 5, 89,
0, 0, 447, 448, 5, 95, 0, 0, 448, 449, 5, 67, 0, 0, 449, 450, 5, 79, 0,
0, 450, 451, 5, 78, 0, 0, 451, 452, 5, 84, 0, 0, 452, 453, 5, 65, 0, 0,
453, 454, 5, 73, 0, 0, 454, 455, 5, 78, 0, 0, 455, 456, 5, 83, 0, 0, 456,
457, 5, 95, 0, 0, 457, 458, 5, 65, 0, 0, 458, 459, 5, 76, 0, 0, 459, 461,
5, 76, 0, 0, 460, 424, 1, 0, 0, 0, 460, 442, 1, 0, 0, 0, 461, 76, 1, 0,
0, 0, 462, 463, 5, 97, 0, 0, 463, 464, 5, 114, 0, 0, 464, 465, 5, 114,
0, 0, 465, 466, 5, 97, 0, 0, 466, 467, 5, 121, 0, 0, 467, 468, 5, 95, 0,
0, 468, 469, 5, 99, 0, 0, 469, 470, 5, 111, 0, 0, 470, 471, 5, 110, 0,
0, 471, 472, 5, 116, 0, 0, 472, 473, 5, 97, 0, 0, 473, 474, 5, 105, 0,
0, 474, 475, 5, 110, 0, 0, 475, 476, 5, 115, 0, 0, 476, 477, 5, 95, 0,
0, 477, 478, 5, 97, 0, 0, 478, 479, 5, 110, 0, 0, 479, 499, 5, 121, 0,
0, 480, 481, 5, 65, 0, 0, 481, 482, 5, 82, 0, 0, 482, 483, 5, 82, 0, 0,
483, 484, 5, 65, 0, 0, 484, 485, 5, 89, 0, 0, 485, 486, 5, 95, 0, 0, 486,
487, 5, 67, 0, 0, 487, 488, 5, 79, 0, 0, 488, 489, 5, 78, 0, 0, 489, 490,
5, 84, 0, 0, 490, 491, 5, 65, 0, 0, 491, 492, 5, 73, 0, 0, 492, 493, 5,
78, 0, 0, 493, 494, 5, 83, 0, 0, 494, 495, 5, 95, 0, 0, 495, 496, 5, 65,
0, 0, 496, 497, 5, 78, 0, 0, 497, 499, 5, 89, 0, 0, 498, 462, 1, 0, 0,
0, 498, 480, 1, 0, 0, 0, 499, 78, 1, 0, 0, 0, 500, 501, 5, 97, 0, 0, 501,
502, 5, 114, 0, 0, 502, 503, 5, 114, 0, 0, 503, 504, 5, 97, 0, 0, 504,
505, 5, 121, 0, 0, 505, 506, 5, 95, 0, 0, 506, 507, 5, 108, 0, 0, 507,
508, 5, 101, 0, 0, 508, 509, 5, 110, 0, 0, 509, 510, 5, 103, 0, 0, 510,
511, 5, 116, 0, 0, 511, 525, 5, 104, 0, 0, 512, 513, 5, 65, 0, 0, 513,
514, 5, 82, 0, 0, 514, 515, 5, 82, 0, 0, 515, 516, 5, 65, 0, 0, 516, 517,
5, 89, 0, 0, 517, 518, 5, 95, 0, 0, 518, 519, 5, 76, 0, 0, 519, 520, 5,
69, 0, 0, 520, 521, 5, 78, 0, 0, 521, 522, 5, 71, 0, 0, 522, 523, 5, 84,
0, 0, 523, 525, 5, 72, 0, 0, 524, 500, 1, 0, 0, 0, 524, 512, 1, 0, 0, 0,
525, 80, 1, 0, 0, 0, 526, 527, 5, 116, 0, 0, 527, 528, 5, 114, 0, 0, 528,
529, 5, 117, 0, 0, 529, 554, 5, 101, 0, 0, 530, 531, 5, 84, 0, 0, 531,
532, 5, 114, 0, 0, 532, 533, 5, 117, 0, 0, 533, 554, 5, 101, 0, 0, 534,
535, 5, 84, 0, 0, 535, 536, 5, 82, 0, 0, 536, 537, 5, 85, 0, 0, 537, 554,
5, 69, 0, 0, 538, 539, 5, 102, 0, 0, 539, 540, 5, 97, 0, 0, 540, 541, 5,
108, 0, 0, 541, 542, 5, 115, 0, 0, 542, 554, 5, 101, 0, 0, 543, 544, 5,
70, 0, 0, 544, 545, 5, 97, 0, 0, 545, 546, 5, 108, 0, 0, 546, 547, 5, 115,
0, 0, 547, 554, 5, 101, 0, 0, 548, 549, 5, 70, 0, 0, 549, 550, 5, 65, 0,
0, 550, 551, 5, 76, 0, 0, 551, 552, 5, 83, 0, 0, 552, 554, 5, 69, 0, 0,
553, 526, 1, 0, 0, 0, 553, 530, 1, 0, 0, 0, 553, 534, 1, 0, 0, 0, 553,
538, 1, 0, 0, 0, 553, 543, 1, 0, 0, 0, 553, 548, 1, 0, 0, 0, 554, 82, 1,
0, 0, 0, 555, 560, 3, 111, 55, 0, 556, 560, 3, 113, 56, 0, 557, 560, 3,
115, 57, 0, 558, 560, 3, 109, 54, 0, 559, 555, 1, 0, 0, 0, 559, 556, 1,
0, 0, 0, 559, 557, 1, 0, 0, 0, 559, 558, 1, 0, 0, 0, 560, 84, 1, 0, 0,
0, 561, 564, 3, 127, 63, 0, 562, 564, 3, 129, 64, 0, 563, 561, 1, 0, 0,
0, 563, 562, 1, 0, 0, 0, 564, 86, 1, 0, 0, 0, 565, 570, 3, 105, 52, 0,
566, 569, 3, 105, 52, 0, 567, 569, 3, 107, 53, 0, 568, 566, 1, 0, 0, 0,
568, 567, 1, 0, 0, 0, 569, 572, 1, 0, 0, 0, 570, 568, 1, 0, 0, 0, 570,
571, 1, 0, 0, 0, 571, 88, 1, 0, 0, 0, 572, 570, 1, 0, 0, 0, 573, 574, 5,
36, 0, 0, 574, 575, 5, 109, 0, 0, 575, 576, 5, 101, 0, 0, 576, 577, 5,
116, 0, 0, 577, 578, 5, 97, 0, 0, 578, 90, 1, 0, 0, 0, 579, 581, 3, 95,
47, 0, 580, 579, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 592, 1, 0, 0, 0,
582, 584, 5, 34, 0, 0, 583, 585, 3, 97, 48, 0, 584, 583, 1, 0, 0, 0, 584,
585, 1, 0, 0, 0, 585, 586, 1, 0, 0, 0, 586, 593, 5, 34, 0, 0, 587, 589,
5, 39, 0, 0, 588, 590, 3, 99, 49, 0, 589, 588, 1, 0, 0, 0, 589, 590, 1,
0, 0, 0, 590, 591, 1, 0, 0, 0, 591, 593, 5, 39, 0, 0, 592, 582, 1, 0, 0,
0, 592, 587, 1, 0, 0, 0, 593, 92, 1, 0, 0, 0, 594, 597, 3, 87, 43, 0, 595,
597, 3, 89, 44, 0, 596, 594, 1, 0, 0, 0, 596, 595, 1, 0, 0, 0, 597, 605,
1, 0, 0, 0, 598, 601, 5, 91, 0, 0, 599, 602, 3, 91, 45, 0, 600, 602, 3,
111, 55, 0, 601, 599, 1, 0, 0, 0, 601, 600, 1, 0, 0, 0, 602, 603, 1, 0,
0, 0, 603, 604, 5, 93, 0, 0, 604, 606, 1, 0, 0, 0, 605, 598, 1, 0, 0, 0,
606, 607, 1, 0, 0, 0, 607, 605, 1, 0, 0, 0, 607, 608, 1, 0, 0, 0, 608,
94, 1, 0, 0, 0, 609, 610, 5, 117, 0, 0, 610, 613, 5, 56, 0, 0, 611, 613,
7, 0, 0, 0, 612, 609, 1, 0, 0, 0, 612, 611, 1, 0, 0, 0, 613, 96, 1, 0,
0, 0, 614, 616, 3, 101, 50, 0, 615, 614, 1, 0, 0, 0, 616, 617, 1, 0, 0,
0, 617, 615, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 98, 1, 0, 0, 0, 619,
621, 3, 103, 51, 0, 620, 619, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 620,
1, 0, 0, 0, 622, 623, 1, 0, 0, 0, 623, 100, 1, 0, 0, 0, 624, 632, 8, 1,
0, 0, 625, 632, 3, 143, 71, 0, 626, 627, 5, 92, 0, 0, 627, 632, 5, 10,
0, 0, 628, 629, 5, 92, 0, 0, 629, 630, 5, 13, 0, 0, 630, 632, 5, 10, 0,
0, 631, 624, 1, 0, 0, 0, 631, 625, 1, 0, 0, 0, 631, 626, 1, 0, 0, 0, 631,
628, 1, 0, 0, 0, 632, 102, 1, 0, 0, 0, 633, 641, 8, 2, 0, 0, 634, 641,
3, 143, 71, 0, 635, 636, 5, 92, 0, 0, 636, 641, 5, 10, 0, 0, 637, 638,
5, 92, 0, 0, 638, 639, 5, 13, 0, 0, 639, 641, 5, 10, 0, 0, 640, 633, 1,
0, 0, 0, 640, 634, 1, 0, 0, 0, 640, 635, 1, 0, 0, 0, 640, 637, 1, 0, 0,
0, 641, 104, 1, 0, 0, 0, 642, 643, 7, 3, 0, 0, 643, 106, 1, 0, 0, 0, 644,
645, 7, 4, 0, 0, 645, 108, 1, 0, 0, 0, 646, 647, 5, 48, 0, 0, 647, 649,
7, 5, 0, 0, 648, 650, 7, 6, 0, 0, 649, 648, 1, 0, 0, 0, 650, 651, 1, 0,
0, 0, 651, 649, 1, 0, 0, 0, 651, 652, 1, 0, 0, 0, 652, 110, 1, 0, 0, 0,
653, 657, 3, 117, 58, 0, 654, 656, 3, 107, 53, 0, 655, 654, 1, 0, 0, 0,
656, 659, 1, 0, 0, 0, 657, 655, 1, 0, 0, 0, 657, 658, 1, 0, 0, 0, 658,
662, 1, 0, 0, 0, 659, 657, 1, 0, 0, 0, 660, 662, 5, 48, 0, 0, 661, 653,
1, 0, 0, 0, 661, 660, 1, 0, 0, 0, 662, 112, 1, 0, 0, 0, 663, 667, 5, 48,
0, 0, 664, 666, 3, 119, 59, 0, 665, 664, 1, 0, 0, 0, 666, 669, 1, 0, 0,
0, 667, 665, 1, 0, 0, 0, 667, 668, 1, 0, 0, 0, 668, 114, 1, 0, 0, 0, 669,
667, 1, 0, 0, 0, 670, 671, 5, 48, 0, 0, 671, 672, 7, 7, 0, 0, 672, 673,
3, 139, 69, 0, 673, 116, 1, 0, 0, 0, 674, 675, 7, 8, 0, 0, 675, 118, 1,
0, 0, 0, 676, 677, 7, 9, 0, 0, 677, 120, 1, 0, 0, 0, 678, 679, 7, 10, 0,
0, 679, 122, 1, 0, 0, 0, 680, 681, 3, 121, 60, 0, 681, 682, 3, 121, 60,
0, 682, 683, 3, 121, 60, 0, 683, 684, 3, 121, 60, 0, 684, 124, 1, 0, 0,
0, 685, 686, 5, 92, 0, 0, 686, 687, 5, 117, 0, 0, 687, 688, 1, 0, 0, 0,
688, 696, 3, 123, 61, 0, 689, 690, 5, 92, 0, 0, 690, 691, 5, 85, 0, 0,
691, 692, 1, 0, 0, 0, 692, 693, 3, 123, 61, 0, 693, 694, 3, 123, 61, 0,
694, 696, 1, 0, 0, 0, 695, 685, 1, 0, 0, 0, 695, 689, 1, 0, 0, 0, 696,
126, 1, 0, 0, 0, 697, 699, 3, 131, 65, 0, 698, 700, 3, 133, 66, 0, 699,
698, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 705, 1, 0, 0, 0, 701, 702,
3, 135, 67, 0, 702, 703, 3, 133, 66, 0, 703, 705, 1, 0, 0, 0, 704, 697,
1, 0, 0, 0, 704, 701, 1, 0, 0, 0, 705, 128, 1, 0, 0, 0, 706, 707, 5, 48,
0, 0, 707, 710, 7, 7, 0, 0, 708, 711, 3, 137, 68, 0, 709, 711, 3, 139,
69, 0, 710, 708, 1, 0, 0, 0, 710, 709, 1, 0, 0, 0, 711, 712, 1, 0, 0, 0,
712, 713, 3, 141, 70, 0, 713, 130, 1, 0, 0, 0, 714, 716, 3, 135, 67, 0,
715, 714, 1, 0, 0, 0, 715, 716, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717,
718, 5, 46, 0, 0, 718, 723, 3, 135, 67, 0, 719, 720, 3, 135, 67, 0, 720,
721, 5, 46, 0, 0, 721, 723, 1, 0, 0, 0, 722, 715, 1, 0, 0, 0, 722, 719,
1, 0, 0, 0, 723, 132, 1, 0, 0, 0, 724, 726, 7, 11, 0, 0, 725, 727, 7, 12,
0, 0, 726, 725, 1, 0, 0, 0, 726, 727, 1, 0, 0, 0, 727, 728, 1, 0, 0, 0,
728, 729, 3, 135, 67, 0, 729, 134, 1, 0, 0, 0, 730, 732, 3, 107, 53, 0,
731, 730, 1, 0, 0, 0, 732, 733, 1, 0, 0, 0, 733, 731, 1, 0, 0, 0, 733,
734, 1, 0, 0, 0, 734, 136, 1, 0, 0, 0, 735, 737, 3, 139, 69, 0, 736, 735,
1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 738, 1, 0, 0, 0, 738, 739, 5, 46,
0, 0, 739, 744, 3, 139, 69, 0, 740, 741, 3, 139, 69, 0, 741, 742, 5, 46,
0, 0, 742, 744, 1, 0, 0, 0, 743, 736, 1, 0, 0, 0, 743, 740, 1, 0, 0, 0,
744, 138, 1, 0, 0, 0, 745, 747, 3, 121, 60, 0, 746, 745, 1, 0, 0, 0, 747,
748, 1, 0, 0, 0, 748, 746, 1, 0, 0, 0, 748, 749, 1, 0, 0, 0, 749, 140,
1, 0, 0, 0, 750, 752, 7, 13, 0, 0, 751, 753, 7, 12, 0, 0, 752, 751, 1,
0, 0, 0, 752, 753, 1, 0, 0, 0, 753, 754, 1, 0, 0, 0, 754, 755, 3, 135,
67, 0, 755, 142, 1, 0, 0, 0, 756, 757, 5, 92, 0, 0, 757, 772, 7, 14, 0,
0, 758, 759, 5, 92, 0, 0, 759, 761, 3, 119, 59, 0, 760, 762, 3, 119, 59,
0, 761, 760, 1, 0, 0, 0, 761, 762, 1, 0, 0, 0, 762, 764, 1, 0, 0, 0, 763,
765, 3, 119, 59, 0, 764, 763, 1, 0, 0, 0, 764, 765, 1, 0, 0, 0, 765, 772,
1, 0, 0, 0, 766, 767, 5, 92, 0, 0, 767, 768, 5, 120, 0, 0, 768, 769, 1,
0, 0, 0, 769, 772, 3, 139, 69, 0, 770, 772, 3, 125, 62, 0, 771, 756, 1,
0, 0, 0, 771, 758, 1, 0, 0, 0, 771, 766, 1, 0, 0, 0, 771, 770, 1, 0, 0,
0, 772, 144, 1, 0, 0, 0, 773, 775, 7, 15, 0, 0, 774, 773, 1, 0, 0, 0, 775,
776, 1, 0, 0, 0, 776, 774, 1, 0, 0, 0, 776, 777, 1, 0, 0, 0, 777, 778,
1, 0, 0, 0, 778, 779, 6, 72, 0, 0, 779, 146, 1, 0, 0, 0, 780, 782, 5, 13,
0, 0, 781, 783, 5, 10, 0, 0, 782, 781, 1, 0, 0, 0, 782, 783, 1, 0, 0, 0,
783, 786, 1, 0, 0, 0, 784, 786, 5, 10, 0, 0, 785, 780, 1, 0, 0, 0, 785,
784, 1, 0, 0, 0, 786, 787, 1, 0, 0, 0, 787, 788, 6, 73, 0, 0, 788, 148,
1, 0, 0, 0, 56, 0, 187, 201, 230, 262, 268, 276, 282, 287, 289, 320, 356,
392, 422, 460, 498, 524, 553, 559, 563, 568, 570, 580, 584, 589, 592, 596,
601, 607, 612, 617, 622, 631, 640, 651, 657, 661, 667, 695, 699, 704, 710,
715, 722, 726, 733, 736, 743, 748, 752, 761, 764, 771, 776, 782, 785, 1,
6, 0, 0,
0, 0, 95, 1, 0, 0, 0, 0, 97, 1, 0, 0, 0, 0, 99, 1, 0, 0, 0, 0, 101, 1,
0, 0, 0, 0, 103, 1, 0, 0, 0, 0, 105, 1, 0, 0, 0, 0, 107, 1, 0, 0, 0, 0,
159, 1, 0, 0, 0, 0, 161, 1, 0, 0, 0, 1, 163, 1, 0, 0, 0, 3, 165, 1, 0,
0, 0, 5, 167, 1, 0, 0, 0, 7, 169, 1, 0, 0, 0, 9, 171, 1, 0, 0, 0, 11, 173,
1, 0, 0, 0, 13, 175, 1, 0, 0, 0, 15, 177, 1, 0, 0, 0, 17, 179, 1, 0, 0,
0, 19, 182, 1, 0, 0, 0, 21, 184, 1, 0, 0, 0, 23, 187, 1, 0, 0, 0, 25, 190,
1, 0, 0, 0, 27, 201, 1, 0, 0, 0, 29, 215, 1, 0, 0, 0, 31, 244, 1, 0, 0,
0, 33, 246, 1, 0, 0, 0, 35, 248, 1, 0, 0, 0, 37, 250, 1, 0, 0, 0, 39, 252,
1, 0, 0, 0, 41, 254, 1, 0, 0, 0, 43, 256, 1, 0, 0, 0, 45, 259, 1, 0, 0,
0, 47, 262, 1, 0, 0, 0, 49, 265, 1, 0, 0, 0, 51, 267, 1, 0, 0, 0, 53, 269,
1, 0, 0, 0, 55, 276, 1, 0, 0, 0, 57, 282, 1, 0, 0, 0, 59, 284, 1, 0, 0,
0, 61, 290, 1, 0, 0, 0, 63, 296, 1, 0, 0, 0, 65, 298, 1, 0, 0, 0, 67, 334,
1, 0, 0, 0, 69, 370, 1, 0, 0, 0, 71, 406, 1, 0, 0, 0, 73, 436, 1, 0, 0,
0, 75, 474, 1, 0, 0, 0, 77, 512, 1, 0, 0, 0, 79, 538, 1, 0, 0, 0, 81, 558,
1, 0, 0, 0, 83, 580, 1, 0, 0, 0, 85, 604, 1, 0, 0, 0, 87, 626, 1, 0, 0,
0, 89, 650, 1, 0, 0, 0, 91, 678, 1, 0, 0, 0, 93, 698, 1, 0, 0, 0, 95, 727,
1, 0, 0, 0, 97, 733, 1, 0, 0, 0, 99, 737, 1, 0, 0, 0, 101, 739, 1, 0, 0,
0, 103, 747, 1, 0, 0, 0, 105, 754, 1, 0, 0, 0, 107, 770, 1, 0, 0, 0, 109,
786, 1, 0, 0, 0, 111, 789, 1, 0, 0, 0, 113, 794, 1, 0, 0, 0, 115, 805,
1, 0, 0, 0, 117, 814, 1, 0, 0, 0, 119, 816, 1, 0, 0, 0, 121, 818, 1, 0,
0, 0, 123, 820, 1, 0, 0, 0, 125, 835, 1, 0, 0, 0, 127, 837, 1, 0, 0, 0,
129, 844, 1, 0, 0, 0, 131, 848, 1, 0, 0, 0, 133, 850, 1, 0, 0, 0, 135,
852, 1, 0, 0, 0, 137, 854, 1, 0, 0, 0, 139, 869, 1, 0, 0, 0, 141, 878,
1, 0, 0, 0, 143, 880, 1, 0, 0, 0, 145, 896, 1, 0, 0, 0, 147, 898, 1, 0,
0, 0, 149, 905, 1, 0, 0, 0, 151, 917, 1, 0, 0, 0, 153, 920, 1, 0, 0, 0,
155, 924, 1, 0, 0, 0, 157, 945, 1, 0, 0, 0, 159, 948, 1, 0, 0, 0, 161,
959, 1, 0, 0, 0, 163, 164, 5, 40, 0, 0, 164, 2, 1, 0, 0, 0, 165, 166, 5,
41, 0, 0, 166, 4, 1, 0, 0, 0, 167, 168, 5, 91, 0, 0, 168, 6, 1, 0, 0, 0,
169, 170, 5, 44, 0, 0, 170, 8, 1, 0, 0, 0, 171, 172, 5, 93, 0, 0, 172,
10, 1, 0, 0, 0, 173, 174, 5, 123, 0, 0, 174, 12, 1, 0, 0, 0, 175, 176,
5, 125, 0, 0, 176, 14, 1, 0, 0, 0, 177, 178, 5, 60, 0, 0, 178, 16, 1, 0,
0, 0, 179, 180, 5, 60, 0, 0, 180, 181, 5, 61, 0, 0, 181, 18, 1, 0, 0, 0,
182, 183, 5, 62, 0, 0, 183, 20, 1, 0, 0, 0, 184, 185, 5, 62, 0, 0, 185,
186, 5, 61, 0, 0, 186, 22, 1, 0, 0, 0, 187, 188, 5, 61, 0, 0, 188, 189,
5, 61, 0, 0, 189, 24, 1, 0, 0, 0, 190, 191, 5, 33, 0, 0, 191, 192, 5, 61,
0, 0, 192, 26, 1, 0, 0, 0, 193, 194, 5, 108, 0, 0, 194, 195, 5, 105, 0,
0, 195, 196, 5, 107, 0, 0, 196, 202, 5, 101, 0, 0, 197, 198, 5, 76, 0,
0, 198, 199, 5, 73, 0, 0, 199, 200, 5, 75, 0, 0, 200, 202, 5, 69, 0, 0,
201, 193, 1, 0, 0, 0, 201, 197, 1, 0, 0, 0, 202, 28, 1, 0, 0, 0, 203, 204,
5, 101, 0, 0, 204, 205, 5, 120, 0, 0, 205, 206, 5, 105, 0, 0, 206, 207,
5, 115, 0, 0, 207, 208, 5, 116, 0, 0, 208, 216, 5, 115, 0, 0, 209, 210,
5, 69, 0, 0, 210, 211, 5, 88, 0, 0, 211, 212, 5, 73, 0, 0, 212, 213, 5,
83, 0, 0, 213, 214, 5, 84, 0, 0, 214, 216, 5, 83, 0, 0, 215, 203, 1, 0,
0, 0, 215, 209, 1, 0, 0, 0, 216, 30, 1, 0, 0, 0, 217, 218, 5, 84, 0, 0,
218, 219, 5, 101, 0, 0, 219, 220, 5, 120, 0, 0, 220, 221, 5, 116, 0, 0,
221, 222, 5, 77, 0, 0, 222, 223, 5, 97, 0, 0, 223, 224, 5, 116, 0, 0, 224,
225, 5, 99, 0, 0, 225, 245, 5, 104, 0, 0, 226, 227, 5, 116, 0, 0, 227,
228, 5, 101, 0, 0, 228, 229, 5, 120, 0, 0, 229, 230, 5, 116, 0, 0, 230,
231, 5, 109, 0, 0, 231, 232, 5, 97, 0, 0, 232, 233, 5, 116, 0, 0, 233,
234, 5, 99, 0, 0, 234, 245, 5, 104, 0, 0, 235, 236, 5, 84, 0, 0, 236, 237,
5, 69, 0, 0, 237, 238, 5, 88, 0, 0, 238, 239, 5, 84, 0, 0, 239, 240, 5,
77, 0, 0, 240, 241, 5, 65, 0, 0, 241, 242, 5, 84, 0, 0, 242, 243, 5, 67,
0, 0, 243, 245, 5, 72, 0, 0, 244, 217, 1, 0, 0, 0, 244, 226, 1, 0, 0, 0,
244, 235, 1, 0, 0, 0, 245, 32, 1, 0, 0, 0, 246, 247, 5, 43, 0, 0, 247,
34, 1, 0, 0, 0, 248, 249, 5, 45, 0, 0, 249, 36, 1, 0, 0, 0, 250, 251, 5,
42, 0, 0, 251, 38, 1, 0, 0, 0, 252, 253, 5, 47, 0, 0, 253, 40, 1, 0, 0,
0, 254, 255, 5, 37, 0, 0, 255, 42, 1, 0, 0, 0, 256, 257, 5, 42, 0, 0, 257,
258, 5, 42, 0, 0, 258, 44, 1, 0, 0, 0, 259, 260, 5, 60, 0, 0, 260, 261,
5, 60, 0, 0, 261, 46, 1, 0, 0, 0, 262, 263, 5, 62, 0, 0, 263, 264, 5, 62,
0, 0, 264, 48, 1, 0, 0, 0, 265, 266, 5, 38, 0, 0, 266, 50, 1, 0, 0, 0,
267, 268, 5, 124, 0, 0, 268, 52, 1, 0, 0, 0, 269, 270, 5, 94, 0, 0, 270,
54, 1, 0, 0, 0, 271, 272, 5, 38, 0, 0, 272, 277, 5, 38, 0, 0, 273, 274,
5, 97, 0, 0, 274, 275, 5, 110, 0, 0, 275, 277, 5, 100, 0, 0, 276, 271,
1, 0, 0, 0, 276, 273, 1, 0, 0, 0, 277, 56, 1, 0, 0, 0, 278, 279, 5, 124,
0, 0, 279, 283, 5, 124, 0, 0, 280, 281, 5, 111, 0, 0, 281, 283, 5, 114,
0, 0, 282, 278, 1, 0, 0, 0, 282, 280, 1, 0, 0, 0, 283, 58, 1, 0, 0, 0,
284, 285, 5, 126, 0, 0, 285, 60, 1, 0, 0, 0, 286, 291, 5, 33, 0, 0, 287,
288, 5, 110, 0, 0, 288, 289, 5, 111, 0, 0, 289, 291, 5, 116, 0, 0, 290,
286, 1, 0, 0, 0, 290, 287, 1, 0, 0, 0, 291, 62, 1, 0, 0, 0, 292, 293, 5,
105, 0, 0, 293, 297, 5, 110, 0, 0, 294, 295, 5, 73, 0, 0, 295, 297, 5,
78, 0, 0, 296, 292, 1, 0, 0, 0, 296, 294, 1, 0, 0, 0, 297, 64, 1, 0, 0,
0, 298, 303, 5, 91, 0, 0, 299, 302, 3, 159, 79, 0, 300, 302, 3, 161, 80,
0, 301, 299, 1, 0, 0, 0, 301, 300, 1, 0, 0, 0, 302, 305, 1, 0, 0, 0, 303,
301, 1, 0, 0, 0, 303, 304, 1, 0, 0, 0, 304, 306, 1, 0, 0, 0, 305, 303,
1, 0, 0, 0, 306, 307, 5, 93, 0, 0, 307, 66, 1, 0, 0, 0, 308, 309, 5, 106,
0, 0, 309, 310, 5, 115, 0, 0, 310, 311, 5, 111, 0, 0, 311, 312, 5, 110,
0, 0, 312, 313, 5, 95, 0, 0, 313, 314, 5, 99, 0, 0, 314, 315, 5, 111, 0,
0, 315, 316, 5, 110, 0, 0, 316, 317, 5, 116, 0, 0, 317, 318, 5, 97, 0,
0, 318, 319, 5, 105, 0, 0, 319, 320, 5, 110, 0, 0, 320, 335, 5, 115, 0,
0, 321, 322, 5, 74, 0, 0, 322, 323, 5, 83, 0, 0, 323, 324, 5, 79, 0, 0,
324, 325, 5, 78, 0, 0, 325, 326, 5, 95, 0, 0, 326, 327, 5, 67, 0, 0, 327,
328, 5, 79, 0, 0, 328, 329, 5, 78, 0, 0, 329, 330, 5, 84, 0, 0, 330, 331,
5, 65, 0, 0, 331, 332, 5, 73, 0, 0, 332, 333, 5, 78, 0, 0, 333, 335, 5,
83, 0, 0, 334, 308, 1, 0, 0, 0, 334, 321, 1, 0, 0, 0, 335, 68, 1, 0, 0,
0, 336, 337, 5, 106, 0, 0, 337, 338, 5, 115, 0, 0, 338, 339, 5, 111, 0,
0, 339, 340, 5, 110, 0, 0, 340, 341, 5, 95, 0, 0, 341, 342, 5, 99, 0, 0,
342, 343, 5, 111, 0, 0, 343, 344, 5, 110, 0, 0, 344, 345, 5, 116, 0, 0,
345, 346, 5, 97, 0, 0, 346, 347, 5, 105, 0, 0, 347, 348, 5, 110, 0, 0,
348, 349, 5, 115, 0, 0, 349, 350, 5, 95, 0, 0, 350, 351, 5, 97, 0, 0, 351,
352, 5, 108, 0, 0, 352, 371, 5, 108, 0, 0, 353, 354, 5, 74, 0, 0, 354,
355, 5, 83, 0, 0, 355, 356, 5, 79, 0, 0, 356, 357, 5, 78, 0, 0, 357, 358,
5, 95, 0, 0, 358, 359, 5, 67, 0, 0, 359, 360, 5, 79, 0, 0, 360, 361, 5,
78, 0, 0, 361, 362, 5, 84, 0, 0, 362, 363, 5, 65, 0, 0, 363, 364, 5, 73,
0, 0, 364, 365, 5, 78, 0, 0, 365, 366, 5, 83, 0, 0, 366, 367, 5, 95, 0,
0, 367, 368, 5, 65, 0, 0, 368, 369, 5, 76, 0, 0, 369, 371, 5, 76, 0, 0,
370, 336, 1, 0, 0, 0, 370, 353, 1, 0, 0, 0, 371, 70, 1, 0, 0, 0, 372, 373,
5, 106, 0, 0, 373, 374, 5, 115, 0, 0, 374, 375, 5, 111, 0, 0, 375, 376,
5, 110, 0, 0, 376, 377, 5, 95, 0, 0, 377, 378, 5, 99, 0, 0, 378, 379, 5,
111, 0, 0, 379, 380, 5, 110, 0, 0, 380, 381, 5, 116, 0, 0, 381, 382, 5,
97, 0, 0, 382, 383, 5, 105, 0, 0, 383, 384, 5, 110, 0, 0, 384, 385, 5,
115, 0, 0, 385, 386, 5, 95, 0, 0, 386, 387, 5, 97, 0, 0, 387, 388, 5, 110,
0, 0, 388, 407, 5, 121, 0, 0, 389, 390, 5, 74, 0, 0, 390, 391, 5, 83, 0,
0, 391, 392, 5, 79, 0, 0, 392, 393, 5, 78, 0, 0, 393, 394, 5, 95, 0, 0,
394, 395, 5, 67, 0, 0, 395, 396, 5, 79, 0, 0, 396, 397, 5, 78, 0, 0, 397,
398, 5, 84, 0, 0, 398, 399, 5, 65, 0, 0, 399, 400, 5, 73, 0, 0, 400, 401,
5, 78, 0, 0, 401, 402, 5, 83, 0, 0, 402, 403, 5, 95, 0, 0, 403, 404, 5,
65, 0, 0, 404, 405, 5, 78, 0, 0, 405, 407, 5, 89, 0, 0, 406, 372, 1, 0,
0, 0, 406, 389, 1, 0, 0, 0, 407, 72, 1, 0, 0, 0, 408, 409, 5, 97, 0, 0,
409, 410, 5, 114, 0, 0, 410, 411, 5, 114, 0, 0, 411, 412, 5, 97, 0, 0,
412, 413, 5, 121, 0, 0, 413, 414, 5, 95, 0, 0, 414, 415, 5, 99, 0, 0, 415,
416, 5, 111, 0, 0, 416, 417, 5, 110, 0, 0, 417, 418, 5, 116, 0, 0, 418,
419, 5, 97, 0, 0, 419, 420, 5, 105, 0, 0, 420, 421, 5, 110, 0, 0, 421,
437, 5, 115, 0, 0, 422, 423, 5, 65, 0, 0, 423, 424, 5, 82, 0, 0, 424, 425,
5, 82, 0, 0, 425, 426, 5, 65, 0, 0, 426, 427, 5, 89, 0, 0, 427, 428, 5,
95, 0, 0, 428, 429, 5, 67, 0, 0, 429, 430, 5, 79, 0, 0, 430, 431, 5, 78,
0, 0, 431, 432, 5, 84, 0, 0, 432, 433, 5, 65, 0, 0, 433, 434, 5, 73, 0,
0, 434, 435, 5, 78, 0, 0, 435, 437, 5, 83, 0, 0, 436, 408, 1, 0, 0, 0,
436, 422, 1, 0, 0, 0, 437, 74, 1, 0, 0, 0, 438, 439, 5, 97, 0, 0, 439,
440, 5, 114, 0, 0, 440, 441, 5, 114, 0, 0, 441, 442, 5, 97, 0, 0, 442,
443, 5, 121, 0, 0, 443, 444, 5, 95, 0, 0, 444, 445, 5, 99, 0, 0, 445, 446,
5, 111, 0, 0, 446, 447, 5, 110, 0, 0, 447, 448, 5, 116, 0, 0, 448, 449,
5, 97, 0, 0, 449, 450, 5, 105, 0, 0, 450, 451, 5, 110, 0, 0, 451, 452,
5, 115, 0, 0, 452, 453, 5, 95, 0, 0, 453, 454, 5, 97, 0, 0, 454, 455, 5,
108, 0, 0, 455, 475, 5, 108, 0, 0, 456, 457, 5, 65, 0, 0, 457, 458, 5,
82, 0, 0, 458, 459, 5, 82, 0, 0, 459, 460, 5, 65, 0, 0, 460, 461, 5, 89,
0, 0, 461, 462, 5, 95, 0, 0, 462, 463, 5, 67, 0, 0, 463, 464, 5, 79, 0,
0, 464, 465, 5, 78, 0, 0, 465, 466, 5, 84, 0, 0, 466, 467, 5, 65, 0, 0,
467, 468, 5, 73, 0, 0, 468, 469, 5, 78, 0, 0, 469, 470, 5, 83, 0, 0, 470,
471, 5, 95, 0, 0, 471, 472, 5, 65, 0, 0, 472, 473, 5, 76, 0, 0, 473, 475,
5, 76, 0, 0, 474, 438, 1, 0, 0, 0, 474, 456, 1, 0, 0, 0, 475, 76, 1, 0,
0, 0, 476, 477, 5, 97, 0, 0, 477, 478, 5, 114, 0, 0, 478, 479, 5, 114,
0, 0, 479, 480, 5, 97, 0, 0, 480, 481, 5, 121, 0, 0, 481, 482, 5, 95, 0,
0, 482, 483, 5, 99, 0, 0, 483, 484, 5, 111, 0, 0, 484, 485, 5, 110, 0,
0, 485, 486, 5, 116, 0, 0, 486, 487, 5, 97, 0, 0, 487, 488, 5, 105, 0,
0, 488, 489, 5, 110, 0, 0, 489, 490, 5, 115, 0, 0, 490, 491, 5, 95, 0,
0, 491, 492, 5, 97, 0, 0, 492, 493, 5, 110, 0, 0, 493, 513, 5, 121, 0,
0, 494, 495, 5, 65, 0, 0, 495, 496, 5, 82, 0, 0, 496, 497, 5, 82, 0, 0,
497, 498, 5, 65, 0, 0, 498, 499, 5, 89, 0, 0, 499, 500, 5, 95, 0, 0, 500,
501, 5, 67, 0, 0, 501, 502, 5, 79, 0, 0, 502, 503, 5, 78, 0, 0, 503, 504,
5, 84, 0, 0, 504, 505, 5, 65, 0, 0, 505, 506, 5, 73, 0, 0, 506, 507, 5,
78, 0, 0, 507, 508, 5, 83, 0, 0, 508, 509, 5, 95, 0, 0, 509, 510, 5, 65,
0, 0, 510, 511, 5, 78, 0, 0, 511, 513, 5, 89, 0, 0, 512, 476, 1, 0, 0,
0, 512, 494, 1, 0, 0, 0, 513, 78, 1, 0, 0, 0, 514, 515, 5, 97, 0, 0, 515,
516, 5, 114, 0, 0, 516, 517, 5, 114, 0, 0, 517, 518, 5, 97, 0, 0, 518,
519, 5, 121, 0, 0, 519, 520, 5, 95, 0, 0, 520, 521, 5, 108, 0, 0, 521,
522, 5, 101, 0, 0, 522, 523, 5, 110, 0, 0, 523, 524, 5, 103, 0, 0, 524,
525, 5, 116, 0, 0, 525, 539, 5, 104, 0, 0, 526, 527, 5, 65, 0, 0, 527,
528, 5, 82, 0, 0, 528, 529, 5, 82, 0, 0, 529, 530, 5, 65, 0, 0, 530, 531,
5, 89, 0, 0, 531, 532, 5, 95, 0, 0, 532, 533, 5, 76, 0, 0, 533, 534, 5,
69, 0, 0, 534, 535, 5, 78, 0, 0, 535, 536, 5, 71, 0, 0, 536, 537, 5, 84,
0, 0, 537, 539, 5, 72, 0, 0, 538, 514, 1, 0, 0, 0, 538, 526, 1, 0, 0, 0,
539, 80, 1, 0, 0, 0, 540, 541, 5, 115, 0, 0, 541, 542, 5, 116, 0, 0, 542,
543, 5, 95, 0, 0, 543, 544, 5, 101, 0, 0, 544, 545, 5, 113, 0, 0, 545,
546, 5, 117, 0, 0, 546, 547, 5, 97, 0, 0, 547, 548, 5, 108, 0, 0, 548,
559, 5, 115, 0, 0, 549, 550, 5, 83, 0, 0, 550, 551, 5, 84, 0, 0, 551, 552,
5, 95, 0, 0, 552, 553, 5, 69, 0, 0, 553, 554, 5, 81, 0, 0, 554, 555, 5,
85, 0, 0, 555, 556, 5, 65, 0, 0, 556, 557, 5, 76, 0, 0, 557, 559, 5, 83,
0, 0, 558, 540, 1, 0, 0, 0, 558, 549, 1, 0, 0, 0, 559, 82, 1, 0, 0, 0,
560, 561, 5, 115, 0, 0, 561, 562, 5, 116, 0, 0, 562, 563, 5, 95, 0, 0,
563, 564, 5, 116, 0, 0, 564, 565, 5, 111, 0, 0, 565, 566, 5, 117, 0, 0,
566, 567, 5, 99, 0, 0, 567, 568, 5, 104, 0, 0, 568, 569, 5, 101, 0, 0,
569, 581, 5, 115, 0, 0, 570, 571, 5, 83, 0, 0, 571, 572, 5, 84, 0, 0, 572,
573, 5, 95, 0, 0, 573, 574, 5, 84, 0, 0, 574, 575, 5, 79, 0, 0, 575, 576,
5, 85, 0, 0, 576, 577, 5, 67, 0, 0, 577, 578, 5, 72, 0, 0, 578, 579, 5,
69, 0, 0, 579, 581, 5, 83, 0, 0, 580, 560, 1, 0, 0, 0, 580, 570, 1, 0,
0, 0, 581, 84, 1, 0, 0, 0, 582, 583, 5, 115, 0, 0, 583, 584, 5, 116, 0,
0, 584, 585, 5, 95, 0, 0, 585, 586, 5, 111, 0, 0, 586, 587, 5, 118, 0,
0, 587, 588, 5, 101, 0, 0, 588, 589, 5, 114, 0, 0, 589, 590, 5, 108, 0,
0, 590, 591, 5, 97, 0, 0, 591, 592, 5, 112, 0, 0, 592, 605, 5, 115, 0,
0, 593, 594, 5, 83, 0, 0, 594, 595, 5, 84, 0, 0, 595, 596, 5, 95, 0, 0,
596, 597, 5, 79, 0, 0, 597, 598, 5, 86, 0, 0, 598, 599, 5, 69, 0, 0, 599,
600, 5, 82, 0, 0, 600, 601, 5, 76, 0, 0, 601, 602, 5, 65, 0, 0, 602, 603,
5, 80, 0, 0, 603, 605, 5, 83, 0, 0, 604, 582, 1, 0, 0, 0, 604, 593, 1,
0, 0, 0, 605, 86, 1, 0, 0, 0, 606, 607, 5, 115, 0, 0, 607, 608, 5, 116,
0, 0, 608, 609, 5, 95, 0, 0, 609, 610, 5, 99, 0, 0, 610, 611, 5, 114, 0,
0, 611, 612, 5, 111, 0, 0, 612, 613, 5, 115, 0, 0, 613, 614, 5, 115, 0,
0, 614, 615, 5, 101, 0, 0, 615, 627, 5, 115, 0, 0, 616, 617, 5, 83, 0,
0, 617, 618, 5, 84, 0, 0, 618, 619, 5, 95, 0, 0, 619, 620, 5, 67, 0, 0,
620, 621, 5, 82, 0, 0, 621, 622, 5, 79, 0, 0, 622, 623, 5, 83, 0, 0, 623,
624, 5, 83, 0, 0, 624, 625, 5, 69, 0, 0, 625, 627, 5, 83, 0, 0, 626, 606,
1, 0, 0, 0, 626, 616, 1, 0, 0, 0, 627, 88, 1, 0, 0, 0, 628, 629, 5, 115,
0, 0, 629, 630, 5, 116, 0, 0, 630, 631, 5, 95, 0, 0, 631, 632, 5, 99, 0,
0, 632, 633, 5, 111, 0, 0, 633, 634, 5, 110, 0, 0, 634, 635, 5, 116, 0,
0, 635, 636, 5, 97, 0, 0, 636, 637, 5, 105, 0, 0, 637, 638, 5, 110, 0,
0, 638, 651, 5, 115, 0, 0, 639, 640, 5, 83, 0, 0, 640, 641, 5, 84, 0, 0,
641, 642, 5, 95, 0, 0, 642, 643, 5, 67, 0, 0, 643, 644, 5, 79, 0, 0, 644,
645, 5, 78, 0, 0, 645, 646, 5, 84, 0, 0, 646, 647, 5, 65, 0, 0, 647, 648,
5, 73, 0, 0, 648, 649, 5, 78, 0, 0, 649, 651, 5, 83, 0, 0, 650, 628, 1,
0, 0, 0, 650, 639, 1, 0, 0, 0, 651, 90, 1, 0, 0, 0, 652, 653, 5, 115, 0,
0, 653, 654, 5, 116, 0, 0, 654, 655, 5, 95, 0, 0, 655, 656, 5, 105, 0,
0, 656, 657, 5, 110, 0, 0, 657, 658, 5, 116, 0, 0, 658, 659, 5, 101, 0,
0, 659, 660, 5, 114, 0, 0, 660, 661, 5, 115, 0, 0, 661, 662, 5, 101, 0,
0, 662, 663, 5, 99, 0, 0, 663, 664, 5, 116, 0, 0, 664, 679, 5, 115, 0,
0, 665, 666, 5, 83, 0, 0, 666, 667, 5, 84, 0, 0, 667, 668, 5, 95, 0, 0,
668, 669, 5, 73, 0, 0, 669, 670, 5, 78, 0, 0, 670, 671, 5, 84, 0, 0, 671,
672, 5, 69, 0, 0, 672, 673, 5, 82, 0, 0, 673, 674, 5, 83, 0, 0, 674, 675,
5, 69, 0, 0, 675, 676, 5, 67, 0, 0, 676, 677, 5, 84, 0, 0, 677, 679, 5,
83, 0, 0, 678, 652, 1, 0, 0, 0, 678, 665, 1, 0, 0, 0, 679, 92, 1, 0, 0,
0, 680, 681, 5, 115, 0, 0, 681, 682, 5, 116, 0, 0, 682, 683, 5, 95, 0,
0, 683, 684, 5, 119, 0, 0, 684, 685, 5, 105, 0, 0, 685, 686, 5, 116, 0,
0, 686, 687, 5, 104, 0, 0, 687, 688, 5, 105, 0, 0, 688, 699, 5, 110, 0,
0, 689, 690, 5, 83, 0, 0, 690, 691, 5, 84, 0, 0, 691, 692, 5, 95, 0, 0,
692, 693, 5, 87, 0, 0, 693, 694, 5, 73, 0, 0, 694, 695, 5, 84, 0, 0, 695,
696, 5, 72, 0, 0, 696, 697, 5, 73, 0, 0, 697, 699, 5, 78, 0, 0, 698, 680,
1, 0, 0, 0, 698, 689, 1, 0, 0, 0, 699, 94, 1, 0, 0, 0, 700, 701, 5, 116,
0, 0, 701, 702, 5, 114, 0, 0, 702, 703, 5, 117, 0, 0, 703, 728, 5, 101,
0, 0, 704, 705, 5, 84, 0, 0, 705, 706, 5, 114, 0, 0, 706, 707, 5, 117,
0, 0, 707, 728, 5, 101, 0, 0, 708, 709, 5, 84, 0, 0, 709, 710, 5, 82, 0,
0, 710, 711, 5, 85, 0, 0, 711, 728, 5, 69, 0, 0, 712, 713, 5, 102, 0, 0,
713, 714, 5, 97, 0, 0, 714, 715, 5, 108, 0, 0, 715, 716, 5, 115, 0, 0,
716, 728, 5, 101, 0, 0, 717, 718, 5, 70, 0, 0, 718, 719, 5, 97, 0, 0, 719,
720, 5, 108, 0, 0, 720, 721, 5, 115, 0, 0, 721, 728, 5, 101, 0, 0, 722,
723, 5, 70, 0, 0, 723, 724, 5, 65, 0, 0, 724, 725, 5, 76, 0, 0, 725, 726,
5, 83, 0, 0, 726, 728, 5, 69, 0, 0, 727, 700, 1, 0, 0, 0, 727, 704, 1,
0, 0, 0, 727, 708, 1, 0, 0, 0, 727, 712, 1, 0, 0, 0, 727, 717, 1, 0, 0,
0, 727, 722, 1, 0, 0, 0, 728, 96, 1, 0, 0, 0, 729, 734, 3, 125, 62, 0,
730, 734, 3, 127, 63, 0, 731, 734, 3, 129, 64, 0, 732, 734, 3, 123, 61,
0, 733, 729, 1, 0, 0, 0, 733, 730, 1, 0, 0, 0, 733, 731, 1, 0, 0, 0, 733,
732, 1, 0, 0, 0, 734, 98, 1, 0, 0, 0, 735, 738, 3, 141, 70, 0, 736, 738,
3, 143, 71, 0, 737, 735, 1, 0, 0, 0, 737, 736, 1, 0, 0, 0, 738, 100, 1,
0, 0, 0, 739, 744, 3, 119, 59, 0, 740, 743, 3, 119, 59, 0, 741, 743, 3,
121, 60, 0, 742, 740, 1, 0, 0, 0, 742, 741, 1, 0, 0, 0, 743, 746, 1, 0,
0, 0, 744, 742, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 102, 1, 0, 0, 0,
746, 744, 1, 0, 0, 0, 747, 748, 5, 36, 0, 0, 748, 749, 5, 109, 0, 0, 749,
750, 5, 101, 0, 0, 750, 751, 5, 116, 0, 0, 751, 752, 5, 97, 0, 0, 752,
104, 1, 0, 0, 0, 753, 755, 3, 109, 54, 0, 754, 753, 1, 0, 0, 0, 754, 755,
1, 0, 0, 0, 755, 766, 1, 0, 0, 0, 756, 758, 5, 34, 0, 0, 757, 759, 3, 111,
55, 0, 758, 757, 1, 0, 0, 0, 758, 759, 1, 0, 0, 0, 759, 760, 1, 0, 0, 0,
760, 767, 5, 34, 0, 0, 761, 763, 5, 39, 0, 0, 762, 764, 3, 113, 56, 0,
763, 762, 1, 0, 0, 0, 763, 764, 1, 0, 0, 0, 764, 765, 1, 0, 0, 0, 765,
767, 5, 39, 0, 0, 766, 756, 1, 0, 0, 0, 766, 761, 1, 0, 0, 0, 767, 106,
1, 0, 0, 0, 768, 771, 3, 101, 50, 0, 769, 771, 3, 103, 51, 0, 770, 768,
1, 0, 0, 0, 770, 769, 1, 0, 0, 0, 771, 779, 1, 0, 0, 0, 772, 775, 5, 91,
0, 0, 773, 776, 3, 105, 52, 0, 774, 776, 3, 125, 62, 0, 775, 773, 1, 0,
0, 0, 775, 774, 1, 0, 0, 0, 776, 777, 1, 0, 0, 0, 777, 778, 5, 93, 0, 0,
778, 780, 1, 0, 0, 0, 779, 772, 1, 0, 0, 0, 780, 781, 1, 0, 0, 0, 781,
779, 1, 0, 0, 0, 781, 782, 1, 0, 0, 0, 782, 108, 1, 0, 0, 0, 783, 784,
5, 117, 0, 0, 784, 787, 5, 56, 0, 0, 785, 787, 7, 0, 0, 0, 786, 783, 1,
0, 0, 0, 786, 785, 1, 0, 0, 0, 787, 110, 1, 0, 0, 0, 788, 790, 3, 115,
57, 0, 789, 788, 1, 0, 0, 0, 790, 791, 1, 0, 0, 0, 791, 789, 1, 0, 0, 0,
791, 792, 1, 0, 0, 0, 792, 112, 1, 0, 0, 0, 793, 795, 3, 117, 58, 0, 794,
793, 1, 0, 0, 0, 795, 796, 1, 0, 0, 0, 796, 794, 1, 0, 0, 0, 796, 797,
1, 0, 0, 0, 797, 114, 1, 0, 0, 0, 798, 806, 8, 1, 0, 0, 799, 806, 3, 157,
78, 0, 800, 801, 5, 92, 0, 0, 801, 806, 5, 10, 0, 0, 802, 803, 5, 92, 0,
0, 803, 804, 5, 13, 0, 0, 804, 806, 5, 10, 0, 0, 805, 798, 1, 0, 0, 0,
805, 799, 1, 0, 0, 0, 805, 800, 1, 0, 0, 0, 805, 802, 1, 0, 0, 0, 806,
116, 1, 0, 0, 0, 807, 815, 8, 2, 0, 0, 808, 815, 3, 157, 78, 0, 809, 810,
5, 92, 0, 0, 810, 815, 5, 10, 0, 0, 811, 812, 5, 92, 0, 0, 812, 813, 5,
13, 0, 0, 813, 815, 5, 10, 0, 0, 814, 807, 1, 0, 0, 0, 814, 808, 1, 0,
0, 0, 814, 809, 1, 0, 0, 0, 814, 811, 1, 0, 0, 0, 815, 118, 1, 0, 0, 0,
816, 817, 7, 3, 0, 0, 817, 120, 1, 0, 0, 0, 818, 819, 7, 4, 0, 0, 819,
122, 1, 0, 0, 0, 820, 821, 5, 48, 0, 0, 821, 823, 7, 5, 0, 0, 822, 824,
7, 6, 0, 0, 823, 822, 1, 0, 0, 0, 824, 825, 1, 0, 0, 0, 825, 823, 1, 0,
0, 0, 825, 826, 1, 0, 0, 0, 826, 124, 1, 0, 0, 0, 827, 831, 3, 131, 65,
0, 828, 830, 3, 121, 60, 0, 829, 828, 1, 0, 0, 0, 830, 833, 1, 0, 0, 0,
831, 829, 1, 0, 0, 0, 831, 832, 1, 0, 0, 0, 832, 836, 1, 0, 0, 0, 833,
831, 1, 0, 0, 0, 834, 836, 5, 48, 0, 0, 835, 827, 1, 0, 0, 0, 835, 834,
1, 0, 0, 0, 836, 126, 1, 0, 0, 0, 837, 841, 5, 48, 0, 0, 838, 840, 3, 133,
66, 0, 839, 838, 1, 0, 0, 0, 840, 843, 1, 0, 0, 0, 841, 839, 1, 0, 0, 0,
841, 842, 1, 0, 0, 0, 842, 128, 1, 0, 0, 0, 843, 841, 1, 0, 0, 0, 844,
845, 5, 48, 0, 0, 845, 846, 7, 7, 0, 0, 846, 847, 3, 153, 76, 0, 847, 130,
1, 0, 0, 0, 848, 849, 7, 8, 0, 0, 849, 132, 1, 0, 0, 0, 850, 851, 7, 9,
0, 0, 851, 134, 1, 0, 0, 0, 852, 853, 7, 10, 0, 0, 853, 136, 1, 0, 0, 0,
854, 855, 3, 135, 67, 0, 855, 856, 3, 135, 67, 0, 856, 857, 3, 135, 67,
0, 857, 858, 3, 135, 67, 0, 858, 138, 1, 0, 0, 0, 859, 860, 5, 92, 0, 0,
860, 861, 5, 117, 0, 0, 861, 862, 1, 0, 0, 0, 862, 870, 3, 137, 68, 0,
863, 864, 5, 92, 0, 0, 864, 865, 5, 85, 0, 0, 865, 866, 1, 0, 0, 0, 866,
867, 3, 137, 68, 0, 867, 868, 3, 137, 68, 0, 868, 870, 1, 0, 0, 0, 869,
859, 1, 0, 0, 0, 869, 863, 1, 0, 0, 0, 870, 140, 1, 0, 0, 0, 871, 873,
3, 145, 72, 0, 872, 874, 3, 147, 73, 0, 873, 872, 1, 0, 0, 0, 873, 874,
1, 0, 0, 0, 874, 879, 1, 0, 0, 0, 875, 876, 3, 149, 74, 0, 876, 877, 3,
147, 73, 0, 877, 879, 1, 0, 0, 0, 878, 871, 1, 0, 0, 0, 878, 875, 1, 0,
0, 0, 879, 142, 1, 0, 0, 0, 880, 881, 5, 48, 0, 0, 881, 884, 7, 7, 0, 0,
882, 885, 3, 151, 75, 0, 883, 885, 3, 153, 76, 0, 884, 882, 1, 0, 0, 0,
884, 883, 1, 0, 0, 0, 885, 886, 1, 0, 0, 0, 886, 887, 3, 155, 77, 0, 887,
144, 1, 0, 0, 0, 888, 890, 3, 149, 74, 0, 889, 888, 1, 0, 0, 0, 889, 890,
1, 0, 0, 0, 890, 891, 1, 0, 0, 0, 891, 892, 5, 46, 0, 0, 892, 897, 3, 149,
74, 0, 893, 894, 3, 149, 74, 0, 894, 895, 5, 46, 0, 0, 895, 897, 1, 0,
0, 0, 896, 889, 1, 0, 0, 0, 896, 893, 1, 0, 0, 0, 897, 146, 1, 0, 0, 0,
898, 900, 7, 11, 0, 0, 899, 901, 7, 12, 0, 0, 900, 899, 1, 0, 0, 0, 900,
901, 1, 0, 0, 0, 901, 902, 1, 0, 0, 0, 902, 903, 3, 149, 74, 0, 903, 148,
1, 0, 0, 0, 904, 906, 3, 121, 60, 0, 905, 904, 1, 0, 0, 0, 906, 907, 1,
0, 0, 0, 907, 905, 1, 0, 0, 0, 907, 908, 1, 0, 0, 0, 908, 150, 1, 0, 0,
0, 909, 911, 3, 153, 76, 0, 910, 909, 1, 0, 0, 0, 910, 911, 1, 0, 0, 0,
911, 912, 1, 0, 0, 0, 912, 913, 5, 46, 0, 0, 913, 918, 3, 153, 76, 0, 914,
915, 3, 153, 76, 0, 915, 916, 5, 46, 0, 0, 916, 918, 1, 0, 0, 0, 917, 910,
1, 0, 0, 0, 917, 914, 1, 0, 0, 0, 918, 152, 1, 0, 0, 0, 919, 921, 3, 135,
67, 0, 920, 919, 1, 0, 0, 0, 921, 922, 1, 0, 0, 0, 922, 920, 1, 0, 0, 0,
922, 923, 1, 0, 0, 0, 923, 154, 1, 0, 0, 0, 924, 926, 7, 13, 0, 0, 925,
927, 7, 12, 0, 0, 926, 925, 1, 0, 0, 0, 926, 927, 1, 0, 0, 0, 927, 928,
1, 0, 0, 0, 928, 929, 3, 149, 74, 0, 929, 156, 1, 0, 0, 0, 930, 931, 5,
92, 0, 0, 931, 946, 7, 14, 0, 0, 932, 933, 5, 92, 0, 0, 933, 935, 3, 133,
66, 0, 934, 936, 3, 133, 66, 0, 935, 934, 1, 0, 0, 0, 935, 936, 1, 0, 0,
0, 936, 938, 1, 0, 0, 0, 937, 939, 3, 133, 66, 0, 938, 937, 1, 0, 0, 0,
938, 939, 1, 0, 0, 0, 939, 946, 1, 0, 0, 0, 940, 941, 5, 92, 0, 0, 941,
942, 5, 120, 0, 0, 942, 943, 1, 0, 0, 0, 943, 946, 3, 153, 76, 0, 944,
946, 3, 139, 69, 0, 945, 930, 1, 0, 0, 0, 945, 932, 1, 0, 0, 0, 945, 940,
1, 0, 0, 0, 945, 944, 1, 0, 0, 0, 946, 158, 1, 0, 0, 0, 947, 949, 7, 15,
0, 0, 948, 947, 1, 0, 0, 0, 949, 950, 1, 0, 0, 0, 950, 948, 1, 0, 0, 0,
950, 951, 1, 0, 0, 0, 951, 952, 1, 0, 0, 0, 952, 953, 6, 79, 0, 0, 953,
160, 1, 0, 0, 0, 954, 956, 5, 13, 0, 0, 955, 957, 5, 10, 0, 0, 956, 955,
1, 0, 0, 0, 956, 957, 1, 0, 0, 0, 957, 960, 1, 0, 0, 0, 958, 960, 5, 10,
0, 0, 959, 954, 1, 0, 0, 0, 959, 958, 1, 0, 0, 0, 960, 961, 1, 0, 0, 0,
961, 962, 6, 80, 0, 0, 962, 162, 1, 0, 0, 0, 63, 0, 201, 215, 244, 276,
282, 290, 296, 301, 303, 334, 370, 406, 436, 474, 512, 538, 558, 580, 604,
626, 650, 678, 698, 727, 733, 737, 742, 744, 754, 758, 763, 766, 770, 775,
781, 786, 791, 796, 805, 814, 825, 831, 835, 841, 869, 873, 878, 884, 889,
896, 900, 907, 910, 917, 922, 926, 935, 938, 945, 950, 956, 959, 1, 6,
0, 0,
}
deserializer := antlr.NewATNDeserializer(nil)
staticData.atn = deserializer.Deserialize(staticData.serializedATN)
@ -513,13 +593,20 @@ const (
PlanLexerArrayContainsAll = 38
PlanLexerArrayContainsAny = 39
PlanLexerArrayLength = 40
PlanLexerBooleanConstant = 41
PlanLexerIntegerConstant = 42
PlanLexerFloatingConstant = 43
PlanLexerIdentifier = 44
PlanLexerMeta = 45
PlanLexerStringLiteral = 46
PlanLexerJSONIdentifier = 47
PlanLexerWhitespace = 48
PlanLexerNewline = 49
PlanLexerSTEuqals = 41
PlanLexerSTTouches = 42
PlanLexerSTOverlaps = 43
PlanLexerSTCrosses = 44
PlanLexerSTContains = 45
PlanLexerSTIntersects = 46
PlanLexerSTWithin = 47
PlanLexerBooleanConstant = 48
PlanLexerIntegerConstant = 49
PlanLexerFloatingConstant = 50
PlanLexerIdentifier = 51
PlanLexerMeta = 52
PlanLexerStringLiteral = 53
PlanLexerJSONIdentifier = 54
PlanLexerWhitespace = 55
PlanLexerNewline = 56
)

1024
internal/parser/planparserv2/generated/plan_parser.go
View File

File diff suppressed because it is too large Load Diff

21
internal/parser/planparserv2/generated/plan_visitor.go
View File

@ -31,6 +31,9 @@ type PlanVisitor interface {
// Visit a parse tree produced by PlanParser#Identifier.
VisitIdentifier(ctx *IdentifierContext) interface{}
// Visit a parse tree produced by PlanParser#STIntersects.
VisitSTIntersects(ctx *STIntersectsContext) interface{}
// Visit a parse tree produced by PlanParser#Like.
VisitLike(ctx *LikeContext) interface{}
@ -52,6 +55,9 @@ type PlanVisitor interface {
// Visit a parse tree produced by PlanParser#Call.
VisitCall(ctx *CallContext) interface{}
// Visit a parse tree produced by PlanParser#STCrosses.
VisitSTCrosses(ctx *STCrossesContext) interface{}
// Visit a parse tree produced by PlanParser#ReverseRange.
VisitReverseRange(ctx *ReverseRangeContext) interface{}
@ -73,12 +79,21 @@ type PlanVisitor interface {
// Visit a parse tree produced by PlanParser#TextMatch.
VisitTextMatch(ctx *TextMatchContext) interface{}
// Visit a parse tree produced by PlanParser#STTouches.
VisitSTTouches(ctx *STTouchesContext) interface{}
// Visit a parse tree produced by PlanParser#STContains.
VisitSTContains(ctx *STContainsContext) interface{}
// Visit a parse tree produced by PlanParser#Term.
VisitTerm(ctx *TermContext) interface{}
// Visit a parse tree produced by PlanParser#JSONContains.
VisitJSONContains(ctx *JSONContainsContext) interface{}
// Visit a parse tree produced by PlanParser#STWithin.
VisitSTWithin(ctx *STWithinContext) interface{}
// Visit a parse tree produced by PlanParser#Range.
VisitRange(ctx *RangeContext) interface{}
@ -103,6 +118,12 @@ type PlanVisitor interface {
// Visit a parse tree produced by PlanParser#BitAnd.
VisitBitAnd(ctx *BitAndContext) interface{}
// Visit a parse tree produced by PlanParser#STEuqals.
VisitSTEuqals(ctx *STEuqalsContext) interface{}
// Visit a parse tree produced by PlanParser#Power.
VisitPower(ctx *PowerContext) interface{}
// Visit a parse tree produced by PlanParser#STOverlaps.
VisitSTOverlaps(ctx *STOverlapsContext) interface{}
}

215
internal/parser/planparserv2/parser_visitor.go
View File

@ -1382,3 +1382,218 @@ func (v *ParserVisitor) VisitTemplateVariable(ctx *parser.TemplateVariableContex
},
}
}
func (v *ParserVisitor) VisitSTEuqals(ctx *parser.STEuqalsContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STEuqals operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText()
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Equals,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTTouches(ctx *parser.STTouchesContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STTouches operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText()
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Touches,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTOverlaps(ctx *parser.STOverlapsContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STOverlaps operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText()
// log.Warn(element)
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Overlaps,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTCrosses(ctx *parser.STCrossesContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STCrosses operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText()
// log.Warn(element)
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Crosses,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTContains(ctx *parser.STContainsContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STContains operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText() // the wkt input
// log.Warn(element)
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Contains,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTIntersects(ctx *parser.STIntersectsContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STIntersects operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText() // the wkt input
// log.Warn(element)
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Intersects,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}
func (v *ParserVisitor) VisitSTWithin(ctx *parser.STWithinContext) interface{} {
childExpr, err := v.translateIdentifier(ctx.Identifier().GetText())
if err != nil {
return err
}
columnInfo := toColumnInfo(childExpr)
if columnInfo == nil ||
(!typeutil.IsGeometryType(columnInfo.GetDataType())) {
return fmt.Errorf(
"STWithin operation are only supported on geometry fields now, got: %s", ctx.GetText())
}
element := ctx.StringLiteral().GetText() // the wkt input
// log.Warn(element)
if err := getError(element); err != nil {
return err
}
expr := &planpb.Expr{
Expr: &planpb.Expr_GisfunctionFilterExpr{
GisfunctionFilterExpr: &planpb.GISFunctionFilterExpr{
ColumnInfo: columnInfo,
WktString: element[1 : len(element)-1],
Op: planpb.GISFunctionFilterExpr_Within,
},
},
}
return &ExprWithType{
expr: expr,
dataType: schemapb.DataType_Bool,
}
}

5
internal/parser/planparserv2/utils.go
View File

@ -152,6 +152,11 @@ func getTargetType(lDataType, rDataType schemapb.DataType) (schemapb.DataType, e
return schemapb.DataType_Int64, nil
}
}
if typeutil.IsGeometryType(lDataType) {
if typeutil.IsGeometryType(rDataType) || typeutil.IsJSONType(rDataType) {
return schemapb.DataType_Geometry, nil
}
}
if typeutil.IsFloatingType(lDataType) {
if typeutil.IsJSONType(rDataType) || typeutil.IsArithmetic(rDataType) {
return schemapb.DataType_Double, nil

17
internal/proto/plan.proto
View File

@ -145,6 +145,22 @@ message JSONContainsExpr {
string template_variable_name = 5;
}
message GISFunctionFilterExpr{
ColumnInfo column_info = 1;
string wkt_string = 2;
enum GISOp {
Invalid = 0;
Equals = 1;
Touches = 2;
Overlaps = 3;
Crosses = 4;
Contains = 5;
Intersects = 6;
Within = 7;
}
GISOp op = 3;
}
message UnaryExpr {
enum UnaryOp {
Invalid = 0;
@ -205,6 +221,7 @@ message Expr {
AlwaysTrueExpr always_true_expr = 12;
JSONContainsExpr json_contains_expr = 13;
CallExpr call_expr = 14;
GISFunctionFilterExpr gisfunction_filter_expr = 15;
};
bool is_template = 20;
}

3
internal/proxy/task_insert_test.go
View File

@ -47,6 +47,7 @@ func TestInsertTask_CheckAligned(t *testing.T) {
float16VectorFieldSchema := &schemapb.FieldSchema{DataType: schemapb.DataType_Float16Vector}
bfloat16VectorFieldSchema := &schemapb.FieldSchema{DataType: schemapb.DataType_BFloat16Vector}
varCharFieldSchema := &schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}
geometryFieldSchema := &schemapb.FieldSchema{DataType: schemapb.DataType_Geometry}
numRows := 20
dim := 128
@ -78,6 +79,7 @@ func TestInsertTask_CheckAligned(t *testing.T) {
float16VectorFieldSchema,
bfloat16VectorFieldSchema,
varCharFieldSchema,
geometryFieldSchema,
},
},
}
@ -97,6 +99,7 @@ func TestInsertTask_CheckAligned(t *testing.T) {
newFloat16VectorFieldData("Float16Vector", numRows, dim),
newBFloat16VectorFieldData("BFloat16Vector", numRows, dim),
newScalarFieldData(varCharFieldSchema, "VarChar", numRows),
newScalarFieldData(geometryFieldSchema, "Geometry", numRows),
}
err = case2.insertMsg.CheckAligned()
assert.NoError(t, err)

1
internal/proxy/task_query.go
View File

@ -545,6 +545,7 @@ func (t *queryTask) PostExecute(ctx context.Context) error {
log.Warn("fail to reduce query result", zap.Error(err))
return err
}
validateGeometryFieldSearchResult(&t.result.FieldsData)
t.result.OutputFields = t.userOutputFields
metrics.ProxyReduceResultLatency.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), metrics.QueryLabel).Observe(float64(tr.RecordSpan().Milliseconds()))

1
internal/proxy/task_search.go
View File

@ -720,6 +720,7 @@ func (t *searchTask) PostExecute(ctx context.Context) error {
}
}
validateGeometryFieldSearchResult(&t.result.Results.FieldsData)
// reduce done, get final result
limit := t.SearchRequest.GetTopk() - t.SearchRequest.GetOffset()
resultSizeInsufficient := false

15
internal/proxy/task_test.go
View File

@ -69,6 +69,7 @@ const (
testBinaryVecField = "bvec"
testFloat16VecField = "f16vec"
testBFloat16VecField = "bf16vec"
testGeometryField = "geometry"
testVecDim = 128
testMaxVarCharLength = 100
)
@ -84,6 +85,7 @@ func genCollectionSchema(collectionName string) *schemapb.CollectionSchema {
testBinaryVecField,
testFloat16VecField,
testBFloat16VecField,
testGeometryField,
testVecDim,
collectionName)
}
@ -226,6 +228,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
func constructCollectionSchemaWithAllType(
boolField, int32Field, int64Field, floatField, doubleField string,
floatVecField, binaryVecField, float16VecField, bfloat16VecField string,
geometryField string,
dim int,
collectionName string,
) *schemapb.CollectionSchema {
@ -339,6 +342,16 @@ func constructCollectionSchemaWithAllType(
IndexParams: nil,
AutoID: false,
}
g := &schemapb.FieldSchema{
FieldID: 0,
Name: geometryField,
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Geometry,
TypeParams: nil,
IndexParams: nil,
AutoID: false,
}
if enableMultipleVectorFields {
return &schemapb.CollectionSchema{
@ -355,6 +368,7 @@ func constructCollectionSchemaWithAllType(
bVec,
f16Vec,
bf16Vec,
g,
},
}
}
@ -371,6 +385,7 @@ func constructCollectionSchemaWithAllType(
d,
fVec,
// bVec,
g,
},
}
}

98
internal/proxy/validate_util.go
View File

@ -5,6 +5,7 @@ import (
"fmt"
"math"
"reflect"
"strings"
"go.uber.org/zap"
@ -17,6 +18,8 @@ import (
"github.com/milvus-io/milvus/pkg/util/parameterutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/typeutil"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkt"
)
type validateUtil struct {
@ -52,6 +55,43 @@ func withMaxCapCheck() validateOption {
}
}
func validateGeometryFieldSearchResult(array *[]*schemapb.FieldData) error {
for idx, fieldData := range *array {
if fieldData.Type == schemapb.DataType_Geometry {
wkbArray := fieldData.GetScalars().GetGeometryData().Data
wktArray := make([][]byte, len(wkbArray))
for i, data := range wkbArray {
geomT, err := wkb.Unmarshal(data)
if err != nil {
log.Warn("translate the wkb format search result into geometry failed")
return err
}
// MaxDecimalDigits set as 6 temporarily
wktStr, err := wkt.Marshal(geomT, wkt.EncodeOptionWithMaxDecimalDigits(6))
if err != nil {
log.Warn("translate the geomery into its wkt failed")
return err
}
wktArray[i] = []byte(wktStr)
}
// modify the field data
(*array)[idx] = &schemapb.FieldData{
Type: fieldData.GetType(),
FieldName: fieldData.GetFieldName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{GeometryData: &schemapb.GeometryArray{Data: wktArray}},
},
},
FieldId: fieldData.GetFieldId(),
IsDynamic: fieldData.GetIsDynamic(),
}
}
}
return nil
}
func (v *validateUtil) apply(opts ...validateOption) {
for _, opt := range opts {
opt(v)
@ -93,6 +133,10 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, helper *typeutil.Sch
if err := v.checkVarCharFieldData(field, fieldSchema); err != nil {
return err
}
case schemapb.DataType_Geometry:
if err := v.checkGeometryFieldData(field, fieldSchema); err != nil {
return err
}
case schemapb.DataType_JSON:
if err := v.checkJSONFieldData(field, fieldSchema); err != nil {
return err
@ -369,6 +413,13 @@ func (v *validateUtil) fillWithNullValue(field *schemapb.FieldData, fieldSchema
}
}
case *schemapb.ScalarField_GeometryData:
if fieldSchema.GetNullable() {
sd.GeometryData.Data, err = fillWithNullValueImpl(sd.GeometryData.Data, field.GetValidData())
if err != nil {
return err
}
}
default:
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
}
@ -468,6 +519,17 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
case *schemapb.ScalarField_GeometryData:
if len(field.GetValidData()) != numRows {
msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
return merr.WrapErrParameterInvalid(numRows, len(field.GetValidData()), msg)
}
defaultValue := fieldSchema.GetDefaultValue().GetBytesData()
sd.GeometryData.Data, err = fillWithDefaultValueImpl(sd.GeometryData.Data, defaultValue, field.GetValidData())
if err != nil {
return err
}
default:
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
}
@ -637,6 +699,42 @@ func (v *validateUtil) checkVarCharFieldData(field *schemapb.FieldData, fieldSch
return nil
}
func (v *validateUtil) checkGeometryFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
geometryArray := field.GetScalars().GetGeometryData().GetData()
if geometryArray == nil {
msg := fmt.Sprintf("geometry field '%v' is illegal, array type mismatch", field.GetFieldName())
return merr.WrapErrParameterInvalid("need geometry array", "got nil", msg)
}
for index, wktdata := range geometryArray {
// ignore parsed geom, the check is during insert task pre execute,so geo data became wkb
// fmt.Println(strings.Trim(string(wktdata), "\""))
geomT, err := wkt.Unmarshal(strings.Trim(string(wktdata), "\""))
if err != nil {
log.Warn("insert invalid Geometry data!! The wkt data has errors", zap.Error(err))
return merr.WrapErrIoFailedReason(err.Error())
}
geometryArray[index], err = wkb.Marshal(geomT, wkb.NDR)
if err != nil {
log.Warn("insert invalid Geometry data!! Transform to wkb failed, has errors", zap.Error(err))
return merr.WrapErrIoFailedReason(err.Error())
}
}
// replace the field data with wkb data array
field = &schemapb.FieldData{
Type: field.GetType(),
FieldName: field.GetFieldName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{GeometryData: &schemapb.GeometryArray{Data: geometryArray}},
},
},
FieldId: field.GetFieldId(),
IsDynamic: field.GetIsDynamic(),
}
return nil
}
func (v *validateUtil) checkJSONFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
jsonArray := field.GetScalars().GetJsonData().GetData()
if jsonArray == nil && fieldSchema.GetDefaultValue() == nil && !fieldSchema.GetNullable() {

21
internal/storage/data_codec.go
View File

@ -367,6 +367,16 @@ func AddFieldDataToPayload(eventWriter *insertEventWriter, dataType schemapb.Dat
return err
}
}
case schemapb.DataType_Geometry:
for i, singleGeometry := range singleData.(*GeometryFieldData).Data {
isValid := true
if len(singleData.(*GeometryFieldData).ValidData) != 0 {
isValid = singleData.(*GeometryFieldData).ValidData[i]
}
if err = eventWriter.AddOneGeometryToPayload(singleGeometry, isValid); err != nil {
return err
}
}
case schemapb.DataType_BinaryVector:
if err = eventWriter.AddBinaryVectorToPayload(singleData.(*BinaryVectorFieldData).Data, singleData.(*BinaryVectorFieldData).Dim); err != nil {
return err
@ -600,6 +610,17 @@ func AddInsertData(dataType schemapb.DataType, data interface{}, insertData *Ins
jsonFieldData.ValidData = append(jsonFieldData.ValidData, validData...)
insertData.Data[fieldID] = jsonFieldData
return len(singleData), nil
case schemapb.DataType_Geometry:
singleData := data.([][]byte)
if fieldData == nil {
fieldData = &GeometryFieldData{Data: make([][]byte, 0, rowNum)}
}
geometryFieldData := fieldData.(*GeometryFieldData)
geometryFieldData.Data = append(geometryFieldData.Data, singleData...)
geometryFieldData.ValidData = append(geometryFieldData.ValidData, validData...)
insertData.Data[fieldID] = geometryFieldData
return len(singleData), nil
case schemapb.DataType_BinaryVector:
singleData := data.([]byte)

3
internal/storage/data_sorter.go
View File

@ -114,6 +114,9 @@ func (ds *DataSorter) Swap(i, j int) {
case schemapb.DataType_JSON:
data := singleData.(*JSONFieldData).Data
data[i], data[j] = data[j], data[i]
case schemapb.DataType_Geometry:
data := singleData.(*GeometryFieldData).Data
data[i], data[j] = data[j], data[i]
case schemapb.DataType_SparseFloatVector:
fieldData := singleData.(*SparseFloatVectorFieldData)
fieldData.Contents[i], fieldData.Contents[j] = fieldData.Contents[j], fieldData.Contents[i]

88
internal/storage/insert_data.go
View File

@ -309,6 +309,16 @@ func NewFieldData(dataType schemapb.DataType, fieldSchema *schemapb.FieldSchema,
data.ValidData = make([]bool, 0, cap)
}
return data, nil
case schemapb.DataType_Geometry:
data := &GeometryFieldData{
Data: make([][]byte, 0, cap),
Nullable: fieldSchema.GetNullable(),
}
if fieldSchema.GetNullable() {
data.ValidData = make([]bool, 0, cap)
}
return data, nil
case schemapb.DataType_Array:
data := &ArrayFieldData{
Data: make([]*schemapb.ScalarField, 0, cap),
@ -386,6 +396,11 @@ type JSONFieldData struct {
ValidData []bool
Nullable bool
}
type GeometryFieldData struct {
Data [][]byte
ValidData []bool
Nullable bool
}
type BinaryVectorFieldData struct {
Data []byte
Dim int
@ -428,6 +443,7 @@ func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
func (data *StringFieldData) RowNum() int { return len(data.Data) }
func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
func (data *GeometryFieldData) RowNum() int { return len(data.Data) }
func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim }
func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim }
func (data *Float16VectorFieldData) RowNum() int { return len(data.Data) / 2 / data.Dim }
@ -507,6 +523,13 @@ func (data *JSONFieldData) GetRow(i int) any {
return data.Data[i]
}
func (data *GeometryFieldData) GetRow(i int) any {
if data.GetNullable() && !data.ValidData[i] {
return nil
}
return data.Data[i]
}
func (data *BinaryVectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim/8 : (i+1)*data.Dim/8]
}
@ -537,6 +560,7 @@ func (data *DoubleFieldData) GetDataRows() any { return data.Data }
func (data *StringFieldData) GetDataRows() any { return data.Data }
func (data *ArrayFieldData) GetDataRows() any { return data.Data }
func (data *JSONFieldData) GetDataRows() any { return data.Data }
func (data *GeometryFieldData) GetDataRows() any { return data.Data }
func (data *BinaryVectorFieldData) GetDataRows() any { return data.Data }
func (data *FloatVectorFieldData) GetDataRows() any { return data.Data }
func (data *Float16VectorFieldData) GetDataRows() any { return data.Data }
@ -714,6 +738,23 @@ func (data *JSONFieldData) AppendRow(row interface{}) error {
return nil
}
func (data *GeometryFieldData) AppendRow(row interface{}) error {
if data.GetNullable() && row == nil {
data.Data = append(data.Data, make([][]byte, 1)...)
data.ValidData = append(data.ValidData, false)
return nil
}
v, ok := row.([]byte)
if !ok {
return merr.WrapErrParameterInvalid("[]byte", row, "Wrong row type")
}
if data.GetNullable() {
data.ValidData = append(data.ValidData, true)
}
data.Data = append(data.Data, v)
return nil
}
func (data *BinaryVectorFieldData) AppendRow(row interface{}) error {
v, ok := row.([]byte)
if !ok || len(v) != data.Dim/8 {
@ -846,6 +887,14 @@ func (data *JSONFieldData) AppendRows(dataRows interface{}, validDataRows interf
return data.AppendValidDataRows(validDataRows)
}
func (data *GeometryFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
err := data.AppendDataRows(dataRows)
if err != nil {
return err
}
return data.AppendValidDataRows(validDataRows)
}
// AppendDataRows appends FLATTEN vectors to field data.
func (data *BinaryVectorFieldData) AppendRows(dataRows interface{}, validDataRows interface{}) error {
err := data.AppendDataRows(dataRows)
@ -980,6 +1029,15 @@ func (data *JSONFieldData) AppendDataRows(rows interface{}) error {
return nil
}
func (data *GeometryFieldData) AppendDataRows(rows interface{}) error {
v, ok := rows.([][]byte)
if !ok {
return merr.WrapErrParameterInvalid("[][]byte", rows, "Wrong rows type")
}
data.Data = append(data.Data, v...)
return nil
}
// AppendDataRows appends FLATTEN vectors to field data.
func (data *BinaryVectorFieldData) AppendDataRows(rows interface{}) error {
v, ok := rows.([]byte)
@ -1164,6 +1222,18 @@ func (data *JSONFieldData) AppendValidDataRows(rows interface{}) error {
return nil
}
func (data *GeometryFieldData) AppendValidDataRows(rows interface{}) error {
if rows == nil {
return nil
}
v, ok := rows.([]bool)
if !ok {
return merr.WrapErrParameterInvalid("[]bool", rows, "Wrong rows type")
}
data.ValidData = append(data.ValidData, v...)
return nil
}
// AppendValidDataRows appends FLATTEN vectors to field data.
func (data *BinaryVectorFieldData) AppendValidDataRows(rows interface{}) error {
if rows != nil {
@ -1282,6 +1352,10 @@ func (data *DoubleFieldData) GetDataType() schemapb.DataType { return schemapb.D
func (data *StringFieldData) GetDataType() schemapb.DataType { return data.DataType }
func (data *ArrayFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_Array }
func (data *JSONFieldData) GetDataType() schemapb.DataType { return schemapb.DataType_JSON }
func (data *GeometryFieldData) GetDataType() schemapb.DataType {
return schemapb.DataType_Geometry
}
func (data *BinaryVectorFieldData) GetDataType() schemapb.DataType {
return schemapb.DataType_BinaryVector
}
@ -1347,6 +1421,15 @@ func (data *JSONFieldData) GetMemorySize() int {
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
}
func (data *GeometryFieldData) GetMemorySize() int {
var size int
// what's the meaning of 16?
for _, val := range data.Data {
size += len(val) + 16
}
return size + binary.Size(data.ValidData) + binary.Size(data.Nullable)
}
func (data *BoolFieldData) GetRowSize(i int) int { return 1 }
func (data *Int8FieldData) GetRowSize(i int) int { return 1 }
func (data *Int16FieldData) GetRowSize(i int) int { return 2 }
@ -1360,6 +1443,7 @@ func (data *Float16VectorFieldData) GetRowSize(i int) int { return data.Dim * 2
func (data *BFloat16VectorFieldData) GetRowSize(i int) int { return data.Dim * 2 }
func (data *StringFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
func (data *JSONFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
func (data *GeometryFieldData) GetRowSize(i int) int { return len(data.Data[i]) + 16 }
func (data *ArrayFieldData) GetRowSize(i int) int {
switch data.ElementType {
case schemapb.DataType_Bool:
@ -1445,3 +1529,7 @@ func (data *ArrayFieldData) GetNullable() bool {
func (data *JSONFieldData) GetNullable() bool {
return data.Nullable
}
func (data *GeometryFieldData) GetNullable() bool {
return data.Nullable
}

2
internal/storage/payload.go
View File

@ -38,6 +38,7 @@ type PayloadWriterInterface interface {
AddOneStringToPayload(msgs string, isValid bool) error
AddOneArrayToPayload(msg *schemapb.ScalarField, isValid bool) error
AddOneJSONToPayload(msg []byte, isValid bool) error
AddOneGeometryToPayload(msg []byte, isValid bool) error
AddBinaryVectorToPayload(binVec []byte, dim int) error
AddFloatVectorToPayload(binVec []float32, dim int) error
AddFloat16VectorToPayload(binVec []byte, dim int) error
@ -65,6 +66,7 @@ type PayloadReaderInterface interface {
GetStringFromPayload() ([]string, []bool, error)
GetArrayFromPayload() ([]*schemapb.ScalarField, []bool, error)
GetJSONFromPayload() ([][]byte, []bool, error)
GetGeometryFromPayload() ([][]byte, []bool, error)
GetBinaryVectorFromPayload() ([]byte, int, error)
GetFloat16VectorFromPayload() ([]byte, int, error)
GetBFloat16VectorFromPayload() ([]byte, int, error)

22
internal/storage/payload_reader.go
View File

@ -98,6 +98,9 @@ func (r *PayloadReader) GetDataFromPayload() (interface{}, []bool, int, error) {
case schemapb.DataType_JSON:
val, validData, err := r.GetJSONFromPayload()
return val, validData, 0, err
case schemapb.DataType_Geometry:
val, validData, err := r.GetGeometryFromPayload()
return val, validData, 0, err
default:
return nil, nil, 0, merr.WrapErrParameterInvalidMsg("unknown type")
}
@ -432,6 +435,25 @@ func (r *PayloadReader) GetJSONFromPayload() ([][]byte, []bool, error) {
return value, nil, nil
}
func (r *PayloadReader) GetGeometryFromPayload() ([][]byte, []bool, error) {
if r.colType != schemapb.DataType_Geometry {
return nil, nil, merr.WrapErrParameterInvalidMsg(fmt.Sprintf("failed to get Geometry from datatype %v", r.colType.String()))
}
if r.nullable {
return readNullableByteAndConvert(r, func(bytes []byte) []byte {
return bytes
})
}
value, err := readByteAndConvert(r, func(bytes parquet.ByteArray) []byte {
return bytes
})
if err != nil {
return nil, nil, err
}
return value, nil, nil
}
func (r *PayloadReader) GetByteArrayDataSet() (*DataSet[parquet.ByteArray, *file.ByteArrayColumnChunkReader], error) {
if r.colType != schemapb.DataType_String && r.colType != schemapb.DataType_VarChar {
return nil, fmt.Errorf("failed to get string from datatype %v", r.colType.String())

44
internal/storage/payload_writer.go
View File

@ -206,6 +206,25 @@ func (w *NativePayloadWriter) AddDataToPayload(data interface{}, validData []boo
isValid = validData[0]
}
return w.AddOneJSONToPayload(val, isValid)
case schemapb.DataType_Geometry:
val, ok := data.([]byte)
if !ok {
return merr.WrapErrParameterInvalidMsg("incorrect data type")
}
isValid := true
if len(validData) > 1 {
return merr.WrapErrParameterInvalidMsg("wrong input length when add data to payload")
}
if len(validData) == 0 && w.nullable {
return merr.WrapErrParameterInvalidMsg("need pass valid_data when nullable==true")
}
if len(validData) == 1 {
if !w.nullable {
return merr.WrapErrParameterInvalidMsg("no need pass valid_data when nullable==false")
}
isValid = validData[0]
}
return w.AddOneGeometryToPayload(val, isValid)
case schemapb.DataType_BinaryVector:
val, ok := data.([]byte)
if !ok {
@ -546,6 +565,29 @@ func (w *NativePayloadWriter) AddOneJSONToPayload(data []byte, isValid bool) err
return nil
}
func (w *NativePayloadWriter) AddOneGeometryToPayload(data []byte, isValid bool) error {
if w.finished {
return errors.New("can't append data to finished geometry payload")
}
if !w.nullable && !isValid {
return merr.WrapErrParameterInvalidMsg("not support null when nullable is false")
}
builder, ok := w.builder.(*array.BinaryBuilder)
if !ok {
return errors.New("failed to cast geometryBuilder")
}
if !isValid {
builder.AppendNull()
} else {
builder.Append(data)
}
return nil
}
func (w *NativePayloadWriter) AddBinaryVectorToPayload(data []byte, dim int) error {
if w.finished {
return errors.New("can't append data to finished binary vector payload")
@ -752,6 +794,8 @@ func milvusDataTypeToArrowType(dataType schemapb.DataType, dim int) arrow.DataTy
return &arrow.BinaryType{}
case schemapb.DataType_JSON:
return &arrow.BinaryType{}
case schemapb.DataType_Geometry:
return &arrow.BinaryType{}
case schemapb.DataType_FloatVector:
return &arrow.FixedSizeBinaryType{
ByteWidth: dim * 4,

20
internal/storage/print_binlog.go
View File

@ -22,6 +22,8 @@ import (
"os"
"github.com/cockroachdb/errors"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkt"
"golang.org/x/exp/mmap"
"google.golang.org/protobuf/proto"
@ -368,6 +370,24 @@ func printPayloadValues(colType schemapb.DataType, reader PayloadReaderInterface
for i, v := range valids {
fmt.Printf("\t\t%d : %v\n", i, v)
}
// print the wkb bytes
case schemapb.DataType_Geometry:
rows, err := reader.GetPayloadLengthFromReader()
if err != nil {
return err
}
val, valids, err := reader.GetGeometryFromPayload()
if err != nil {
return err
}
for i := 0; i < rows; i++ {
geomT, _ := wkb.Unmarshal(val[i])
wktStr, _ := wkt.Marshal(geomT)
fmt.Printf("\t\t%d : %s\n", i, wktStr)
}
for i, v := range valids {
fmt.Printf("\t\t%d : %v\n", i, v)
}
case schemapb.DataType_SparseFloatVector:
sparseData, _, err := reader.GetSparseFloatVectorFromPayload()
if err != nil {

21
internal/storage/print_binlog_test.go
View File

@ -210,6 +210,13 @@ func TestPrintBinlogFiles(t *testing.T) {
{Key: common.DimKey, Value: "4"},
},
},
{
FieldID: 113,
Name: "field_geometry",
IsPrimaryKey: false,
Description: "description_15",
DataType: schemapb.DataType_Geometry,
},
},
},
}
@ -268,6 +275,13 @@ func TestPrintBinlogFiles(t *testing.T) {
Data: []byte("12345678"),
Dim: 4,
},
113: &GeometryFieldData{
Data: [][]byte{
// POINT (30.123 -10.456) and LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)
{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40},
{0x01, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40, 0x03, 0xA6, 0xB4, 0xA6, 0xA4, 0xD2, 0xC5, 0xC0, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A},
},
},
},
}
@ -325,6 +339,13 @@ func TestPrintBinlogFiles(t *testing.T) {
Data: []byte("abcdefgh"),
Dim: 4,
},
113: &GeometryFieldData{
Data: [][]byte{
// POINT (30.123 -10.456) and LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)
{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40},
{0x01, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40, 0x03, 0xA6, 0xB4, 0xA6, 0xA4, 0xD2, 0xC5, 0xC0, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A},
},
},
},
}
firstBlobs, err := insertCodec.Serialize(1, 1, insertDataFirst)

1
internal/storage/serde.go
View File

@ -398,6 +398,7 @@ var serdeMap = func() map[schemapb.DataType]serdeEntry {
m[schemapb.DataType_Array] = byteEntry
m[schemapb.DataType_JSON] = byteEntry
m[schemapb.DataType_Geometry] = byteEntry
fixedSizeDeserializer := func(a arrow.Array, i int) (any, bool) {
if a.IsNull(i) {

22
internal/storage/utils.go
View File

@ -673,6 +673,13 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
Data: lo.Map(srcData, func(v []byte, _ int) []byte { return v }),
ValidData: lo.Map(validData, func(v bool, _ int) bool { return v }),
}
case schemapb.DataType_Geometry:
srcData := srcField.GetScalars().GetGeometryData().GetData()
validData := srcField.GetValidData()
fieldData = &GeometryFieldData{
Data: lo.Map(srcData, func(v []byte, _ int) []byte { return v }),
ValidData: lo.Map(validData, func(v bool, _ int) bool { return v }),
}
default:
return nil, merr.WrapErrServiceInternal("data type not handled", field.GetDataType().String())
@ -1193,6 +1200,21 @@ func TransferInsertDataToInsertRecord(insertData *InsertData) (*segcorepb.Insert
},
ValidData: rawData.ValidData,
}
case *GeometryFieldData:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldId: fieldID,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: rawData.Data,
},
},
},
},
ValidData: rawData.ValidData,
}
case *FloatVectorFieldData:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_FloatVector,

42
internal/util/importutilv2/parquet/field_reader.go
View File

@ -150,6 +150,12 @@ func (c *FieldReader) Next(count int64) (any, any, error) {
}
data, err := ReadJSONData(c, count)
return data, nil, err
case schemapb.DataType_Geometry:
if c.field.GetNullable() {
return ReadNullableGeometryData(c, count)
}
data, err := ReadGeometryData(c, count)
return data, nil, err
case schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector, schemapb.DataType_BFloat16Vector:
// vector not support default_value
if c.field.GetNullable() {
@ -620,6 +626,42 @@ func ReadNullableJSONData(pcr *FieldReader, count int64) (any, []bool, error) {
return byteArr, validData, nil
}
func ReadNullableGeometryData(pcr *FieldReader, count int64) (any, []bool, error) {
// Geometry field read data from string array Parquet
data, validData, err := ReadNullableStringData(pcr, count)
if err != nil {
return nil, nil, err
}
if data == nil {
return nil, nil, nil
}
byteArr := make([][]byte, 0)
for i, str := range data.([]string) {
if !validData[i] {
byteArr = append(byteArr, []byte(nil))
continue
}
byteArr = append(byteArr, []byte(str))
}
return byteArr, validData, nil
}
func ReadGeometryData(pcr *FieldReader, count int64) (any, error) {
// Geometry field read data from string array Parquet
data, err := ReadStringData(pcr, count)
if err != nil {
return nil, err
}
if data == nil {
return nil, nil
}
byteArr := make([][]byte, 0)
for _, str := range data.([]string) {
byteArr = append(byteArr, []byte(str))
}
return byteArr, nil
}
func ReadBinaryData(pcr *FieldReader, count int64) (any, error) {
dataType := pcr.field.GetDataType()
chunked, err := pcr.columnReader.NextBatch(count)

2
internal/util/importutilv2/parquet/reader_test.go
View File

@ -426,6 +426,7 @@ func (s *ReaderSuite) TestReadScalarFields() {
s.run(schemapb.DataType_String, schemapb.DataType_None, false, 0)
s.run(schemapb.DataType_VarChar, schemapb.DataType_None, false, 0)
s.run(schemapb.DataType_JSON, schemapb.DataType_None, false, 0)
s.run(schemapb.DataType_Geometry, schemapb.DataType_None, false, 0)
s.run(schemapb.DataType_Array, schemapb.DataType_Bool, false, 0)
s.run(schemapb.DataType_Array, schemapb.DataType_Int8, false, 0)
@ -464,6 +465,7 @@ func (s *ReaderSuite) TestReadScalarFields() {
s.run(schemapb.DataType_String, schemapb.DataType_None, true, 100)
s.run(schemapb.DataType_VarChar, schemapb.DataType_None, true, 100)
s.run(schemapb.DataType_JSON, schemapb.DataType_None, true, 100)
s.run(schemapb.DataType_Geometry, schemapb.DataType_None, true, 100)
s.run(schemapb.DataType_Array, schemapb.DataType_Bool, true, 100)
s.run(schemapb.DataType_Array, schemapb.DataType_Int8, true, 100)

2
internal/util/importutilv2/parquet/util.go
View File

@ -175,6 +175,8 @@ func convertToArrowDataType(field *schemapb.FieldSchema, isArray bool) (arrow.Da
return &arrow.StringType{}, nil
case schemapb.DataType_JSON:
return &arrow.StringType{}, nil
case schemapb.DataType_Geometry:
return &arrow.StringType{}, nil
case schemapb.DataType_Array:
elemType, err := convertToArrowDataType(field, true)
if err != nil {

12
internal/util/testutil/test_util.go
View File

@ -175,6 +175,9 @@ func CreateInsertData(schema *schemapb.CollectionSchema, rows int, nullPercent .
insertData.Data[f.FieldID].AppendDataRows(testutils.GenerateStringArray(rows))
case schemapb.DataType_JSON:
insertData.Data[f.FieldID].AppendDataRows(testutils.GenerateJSONArray(rows))
case schemapb.DataType_Geometry:
// wkb bytes array
insertData.Data[f.FieldID].AppendDataRows(testutils.GenerateGeometryArray(rows))
case schemapb.DataType_Array:
switch f.GetElementType() {
case schemapb.DataType_Bool:
@ -427,6 +430,14 @@ func BuildArrayData(schema *schemapb.CollectionSchema, insertData *storage.Inser
return string(bs)
}), validData)
columns = append(columns, builder.NewStringArray())
case schemapb.DataType_Geometry:
builder := array.NewStringBuilder(mem)
wkbData := insertData.Data[fieldID].(*storage.GeometryFieldData).Data
validData := insertData.Data[fieldID].(*storage.GeometryFieldData).ValidData
builder.AppendValues(lo.Map(wkbData, func(bs []byte, _ int) string {
return string(bs)
}), validData)
columns = append(columns, builder.NewStringArray())
case schemapb.DataType_Array:
data := insertData.Data[fieldID].(*storage.ArrayFieldData).Data
validData := insertData.Data[fieldID].(*storage.ArrayFieldData).ValidData
@ -582,6 +593,7 @@ func BuildArrayData(schema *schemapb.CollectionSchema, insertData *storage.Inser
columns = append(columns, builder.NewListArray())
}
}
}
return columns, nil
}

5
internal/util/typeutil/schema.go
View File

@ -62,6 +62,11 @@ func ConvertToArrowSchema(fields []*schemapb.FieldSchema) (*arrow.Schema, error)
Name: field.Name,
Type: arrow.BinaryTypes.Binary,
})
case schemapb.DataType_Geometry:
arrowFields = append(arrowFields, arrow.Field{
Name: field.Name,
Type: arrow.BinaryTypes.Binary,
})
case schemapb.DataType_BinaryVector:
dim, err := storage.GetDimFromParams(field.TypeParams)
if err != nil {

2
internal/util/typeutil/storage.go
View File

@ -77,6 +77,8 @@ func BuildRecord(b *array.RecordBuilder, data *storage.InsertData, fields []*sch
}
case schemapb.DataType_JSON:
fBuilder.(*array.BinaryBuilder).AppendValues(data.Data[field.FieldID].(*storage.JSONFieldData).Data, nil)
case schemapb.DataType_Geometry:
fBuilder.(*array.BinaryBuilder).AppendValues(data.Data[field.FieldID].(*storage.GeometryFieldData).Data, nil)
case schemapb.DataType_BinaryVector:
vecData := data.Data[field.FieldID].(*storage.BinaryVectorFieldData)
for i := 0; i < len(vecData.Data); i += vecData.Dim / 8 {

1
pkg/go.mod
View File

@ -33,6 +33,7 @@ require (
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
github.com/tidwall/gjson v1.17.0
github.com/tikv/client-go/v2 v2.0.4
github.com/twpayne/go-geom v1.4.0
github.com/uber/jaeger-client-go v2.30.0+incompatible
github.com/x448/float16 v0.8.4
go.etcd.io/etcd/api/v3 v3.5.5

34
pkg/go.sum
View File

@ -44,15 +44,23 @@ github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwR
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/AthenZ/athenz v1.10.39 h1:mtwHTF/v62ewY2Z5KWhuZgVXftBej1/Tn80zx4DcawY=
github.com/AthenZ/athenz v1.10.39/go.mod h1:3Tg8HLsiQZp81BJY58JBeU2BR6B/H4/0MQGfCwhHNEA=
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno=
github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo=
github.com/DATA-DOG/go-sqlmock v1.3.2 h1:2L2f5t3kKnCLxnClDD/PrDfExFFa1wjESgxHG/B1ibo=
github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo=
github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
github.com/Masterminds/goutils v1.1.0/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y=
github.com/Masterminds/sprig v2.22.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
github.com/SimFG/expr v0.0.0-20241029080923-21854792f532 h1:fxpBc+wl14Ky70AVgCJ4muWzAzXcxoerlRxA1F69S0M=
@ -90,6 +98,7 @@ github.com/bits-and-blooms/bitset v1.4.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edY
github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs=
github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw=
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
@ -131,6 +140,7 @@ github.com/confluentinc/confluent-kafka-go v1.9.1 h1:L3aW6KvTyrq/+BOMnDm9xJylhAE
github.com/confluentinc/confluent-kafka-go v1.9.1/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo=
github.com/containerd/cgroups/v3 v3.0.3 h1:S5ByHZ/h9PMe5IOQoN7E+nMc2UcLEM/V48DGDJ9kip0=
github.com/containerd/cgroups/v3 v3.0.3/go.mod h1:8HBe7V3aWGLFPd/k03swSIsGjZhHI2WzJmticMgVuz0=
github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@ -159,6 +169,7 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA=
github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0=
github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
@ -328,6 +339,7 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe
github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
@ -382,11 +394,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE=
github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
@ -438,6 +452,7 @@ github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLA
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@ -456,6 +471,8 @@ github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b h1:xYEM2oBUhBE
github.com/kris-nova/lolgopher v0.0.0-20180921204813-313b3abb0d9b/go.mod h1:V0HF/ZBlN86HqewcDC/cVxMmYDiRukWjSrgKLUAn9Js=
github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/linkedin/goavro v2.1.0+incompatible/go.mod h1:bBCwI2eGYpUI/4820s67MElg9tdeLbINjLjiM2xZFYM=
github.com/linkedin/goavro/v2 v2.10.0/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
github.com/linkedin/goavro/v2 v2.10.1/go.mod h1:UgQUb2N/pmueQYH9bfqFioWxzYCZXSfF8Jw03O5sjqA=
@ -500,6 +517,7 @@ github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4M
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@ -509,6 +527,7 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@ -548,11 +567,15 @@ github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0=
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/ory/dockertest/v3 v3.6.0/go.mod h1:4ZOpj8qBUmh8fcBSVzkH2bws2s91JdGvHUqan4GHEuQ=
github.com/panjf2000/ants/v2 v2.7.2 h1:2NUt9BaZFO5kQzrieOmK/wdb/tQ/K+QHaxN8sOgD63U=
github.com/panjf2000/ants/v2 v2.7.2/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
@ -651,6 +674,7 @@ github.com/shirou/gopsutil/v3 v3.22.9 h1:yibtJhIVEMcdw+tCTbOPiF1VcsuDeTE4utJ8Dm4
github.com/shirou/gopsutil/v3 v3.22.9/go.mod h1:bBYl1kjgEJpWpxeHmLI+dVHWtyAwfcmSBLDsp2TNT8A=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
@ -733,6 +757,11 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4
github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
github.com/twmb/murmur3 v1.1.3 h1:D83U0XYKcHRYwYIpBKf3Pks91Z0Byda/9SJ8B6EMRcA=
github.com/twmb/murmur3 v1.1.3/go.mod h1:Qq/R7NUyOfr65zD+6Q5IHKsJLwP7exErjN6lyyq3OSQ=
github.com/twpayne/go-geom v1.4.0 h1:gtI+ClrojsGDl0SXXNAwPUtPxanyodQzXg8mliV3d/w=
github.com/twpayne/go-geom v1.4.0/go.mod h1:k/zktXdL+qnA6OgKsdEGUTA17jbQ2ZPTUa3CCySuGpE=
github.com/twpayne/go-kml v1.5.2/go.mod h1:kz8jAiIz6FIdU2Zjce9qGlVtgFYES9vt7BTPBHf5jl4=
github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU=
github.com/twpayne/go-waypoint v0.0.0-20200706203930-b263a7f6e4e8/go.mod h1:qj5pHncxKhu9gxtZEYWypA/z097sxhFlbTyOyt9gcnU=
github.com/uber/jaeger-client-go v2.30.0+incompatible h1:D6wyKGCecFaSRUpo8lCVbaOOb6ThwMmTEbhRwtKR97o=
github.com/uber/jaeger-client-go v2.30.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
@ -856,6 +885,7 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@ -922,6 +952,7 @@ golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191003171128-d98b1b443823/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@ -1009,6 +1040,7 @@ golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200121082415-34d275377bf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -1091,6 +1123,7 @@ golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@ -1310,6 +1343,7 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

2
pkg/util/funcutil/func.go
View File

@ -355,6 +355,8 @@ func GetNumRowOfFieldDataWithSchema(fieldData *schemapb.FieldData, helper *typeu
fieldNumRows = getNumRowsOfScalarField(fieldData.GetScalars().GetArrayData().GetData())
case schemapb.DataType_JSON:
fieldNumRows = getNumRowsOfScalarField(fieldData.GetScalars().GetJsonData().GetData())
case schemapb.DataType_Geometry:
fieldNumRows = getNumRowsOfScalarField(fieldData.GetScalars().GetGeometryData().GetData())
case schemapb.DataType_FloatVector:
dim := fieldData.GetVectors().GetDim()
fieldNumRows, err = GetNumRowsOfFloatVectorField(fieldData.GetVectors().GetFloatVector().GetData(), dim)

103
pkg/util/testutils/gen_data.go
View File

@ -26,6 +26,8 @@ import (
"strconv"
"strings"
"github.com/twpayne/go-geom/encoding/wkb"
"github.com/twpayne/go-geom/encoding/wkt"
"github.com/x448/float16"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
@ -149,6 +151,55 @@ func GenerateJSONArray(numRows int) [][]byte {
return ret
}
func GenerateGeometryArray(numRows int) [][]byte {
ret := make([][]byte, 0, numRows)
const (
point = "POINT (30.123 -10.456)"
linestring = "LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)"
polygon = "POLYGON ((30.123 -10.456, 40.678 40.890, 20.345 40.567, 10.123 20.456, 30.123 -10.456))"
multipoint = "MULTIPOINT ((10.111 40.222), (40.333 30.444), (20.555 20.666), (30.777 10.888))"
multilinestring = "MULTILINESTRING ((10.111 10.222, 20.333 20.444), (15.555 15.666, 25.777 25.888), (-30.999 20.000, 40.111 30.222))"
multipolygon = "MULTIPOLYGON (((30.123 -10.456, 40.678 40.890, 20.345 40.567, 10.123 20.456, 30.123 -10.456)),((15.123 5.456, 25.678 5.890, 25.345 15.567, 15.123 15.456, 15.123 5.456)))"
)
wkt_array := [6]string{point, linestring, polygon, multipoint, multilinestring, multipolygon}
for i := 0; i < numRows; i++ {
// data of wkt string bytes ,consider to be process by proxy
if i == numRows-1 {
geomT, _ := wkt.Unmarshal("POINT (-84.036 39.997)") // add a special point finally for test
wkbdata, _ := wkb.Marshal(geomT, wkb.NDR)
ret = append(ret, wkbdata)
continue
}
geomT, _ := wkt.Unmarshal(wkt_array[i%6])
wkbdata, _ := wkb.Marshal(geomT, wkb.NDR)
ret = append(ret, wkbdata)
}
return ret
}
func GenerateGeometryWktArray(numRows int) [][]byte {
ret := make([][]byte, 0, numRows)
const (
point = "POINT (30.123 -10.456)"
linestring = "LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)"
polygon = "POLYGON ((30.123 -10.456, 40.678 40.890, 20.345 40.567, 10.123 20.456, 30.123 -10.456))"
multipoint = "MULTIPOINT ((10.111 40.222), (40.333 30.444), (20.555 20.666), (30.777 10.888))"
multilinestring = "MULTILINESTRING ((10.111 10.222, 20.333 20.444), (15.555 15.666, 25.777 25.888), (-30.999 20.000, 40.111 30.222))"
multipolygon = "MULTIPOLYGON (((30.123 -10.456, 40.678 40.890, 20.345 40.567, 10.123 20.456, 30.123 -10.456)),((15.123 5.456, 25.678 5.890, 25.345 15.567, 15.123 15.456, 15.123 5.456)))"
)
wkt_array := [6]string{point, linestring, polygon, multipoint, multilinestring, multipolygon}
for i := 0; i < numRows; i++ {
// data of wkt string bytes ,consider to be process by proxy
if i == numRows-1 {
ret = append(ret, []byte("POINT (-84.036 39.997)"))
continue
}
ret = append(ret, []byte(wkt_array[i%6]))
}
return ret
}
func GenerateArrayOfBoolArray(numRows int) []*schemapb.ScalarField {
ret := make([]*schemapb.ScalarField, 0, numRows)
for i := 0; i < numRows; i++ {
@ -715,6 +766,54 @@ func NewArrayFieldDataWithValue(fieldName string, fieldValue interface{}) *schem
}
}
func NewGeometryFieldData(fieldName string, numRows int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: GenerateGeometryArray(numRows),
},
},
},
},
}
}
func NewGeometryFieldDataWktFormat(fieldName string, numRows int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: GenerateGeometryWktArray(numRows),
},
},
},
},
}
}
func NewGeometryFieldDataWithValue(fieldName string, fieldValue interface{}) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: fieldValue.([][]byte),
},
},
},
},
}
}
func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
@ -882,6 +981,8 @@ func GenerateScalarFieldData(dType schemapb.DataType, fieldName string, numRows
return NewArrayFieldData(fieldName, numRows)
case schemapb.DataType_JSON:
return NewJSONFieldData(fieldName, numRows)
case schemapb.DataType_Geometry:
return NewGeometryFieldData(fieldName, numRows)
default:
panic("unsupported data type")
}
@ -912,6 +1013,8 @@ func GenerateScalarFieldDataWithValue(dType schemapb.DataType, fieldName string,
fieldData = NewArrayFieldDataWithValue(fieldName, fieldValue)
case schemapb.DataType_JSON:
fieldData = NewJSONFieldDataWithValue(fieldName, fieldValue)
case schemapb.DataType_Geometry:
fieldData = NewGeometryFieldDataWithValue(fieldName, fieldValue)
default:
panic("unsupported data type")
}

16
pkg/util/typeutil/gen_empty_field_data.go
View File

@ -123,6 +123,20 @@ func genEmptyJSONFieldData(field *schemapb.FieldSchema) *schemapb.FieldData {
}
}
func genEmptyGeometryFieldData(field *schemapb.FieldSchema) *schemapb.FieldData {
return &schemapb.FieldData{
Type: field.GetDataType(),
FieldName: field.GetName(),
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{GeometryData: &schemapb.GeometryArray{Data: nil}},
},
},
FieldId: field.GetFieldID(),
IsDynamic: field.GetIsDynamic(),
}
}
func genEmptyBinaryVectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dim, err := GetDim(field)
if err != nil {
@ -246,6 +260,8 @@ func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error)
return genEmptyArrayFieldData(field), nil
case schemapb.DataType_JSON:
return genEmptyJSONFieldData(field), nil
case schemapb.DataType_Geometry:
return genEmptyGeometryFieldData(field), nil
case schemapb.DataType_BinaryVector:
return genEmptyBinaryVectorFieldData(field)
case schemapb.DataType_FloatVector:

48
pkg/util/typeutil/schema.go
View File

@ -82,7 +82,7 @@ func getVarFieldLength(fieldSchema *schemapb.FieldSchema, policy getVariableFiel
default:
return 0, fmt.Errorf("unrecognized getVariableFieldLengthPolicy %v", policy)
}
case schemapb.DataType_Array, schemapb.DataType_JSON:
case schemapb.DataType_Array, schemapb.DataType_JSON, schemapb.DataType_Geometry:
return DynamicFieldMaxLength, nil
default:
return 0, fmt.Errorf("field %s is not a variable-length type", fieldSchema.DataType.String())
@ -114,7 +114,7 @@ func estimateSizeBy(schema *schemapb.CollectionSchema, policy getVariableFieldLe
res += 4
case schemapb.DataType_Int64, schemapb.DataType_Double:
res += 8
case schemapb.DataType_VarChar, schemapb.DataType_Array, schemapb.DataType_JSON:
case schemapb.DataType_VarChar, schemapb.DataType_Array, schemapb.DataType_JSON, schemapb.DataType_Geometry: // geo wkb max len
maxLengthPerRow, err := getVarFieldLength(fs, policy)
if err != nil {
return 0, err
@ -196,6 +196,10 @@ func CalcColumnSize(column *schemapb.FieldData) int {
for _, str := range column.GetScalars().GetJsonData().GetData() {
res += len(str)
}
case schemapb.DataType_Geometry:
for _, str := range column.GetScalars().GetGeometryData().GetData() {
res += len(str)
}
default:
panic("Unknown data type:" + column.Type.String())
}
@ -233,6 +237,11 @@ func EstimateEntitySize(fieldsData []*schemapb.FieldData, rowOffset int) (int, e
return 0, fmt.Errorf("offset out range of field datas")
}
res += len(fs.GetScalars().GetJsonData().GetData()[rowOffset])
case schemapb.DataType_Geometry:
if rowOffset >= len(fs.GetScalars().GetGeometryData().GetData()) {
return 0, fmt.Errorf("offset out range of field datas")
}
res += len(fs.GetScalars().GetGeometryData().GetData()[rowOffset])
case schemapb.DataType_BinaryVector:
res += int(fs.GetVectors().GetDim())
case schemapb.DataType_FloatVector:
@ -503,6 +512,10 @@ func IsJSONType(dataType schemapb.DataType) bool {
return dataType == schemapb.DataType_JSON
}
func IsGeometryType(dataType schemapb.DataType) bool {
return dataType == schemapb.DataType_Geometry
}
func IsArrayType(dataType schemapb.DataType) bool {
return dataType == schemapb.DataType_Array
}
@ -543,7 +556,7 @@ func IsStringType(dataType schemapb.DataType) bool {
}
func IsVariableDataType(dataType schemapb.DataType) bool {
return IsStringType(dataType) || IsArrayType(dataType) || IsJSONType(dataType)
return IsStringType(dataType) || IsArrayType(dataType) || IsJSONType(dataType) || IsGeometryType(dataType)
}
func IsPrimitiveType(dataType schemapb.DataType) bool {
@ -610,6 +623,12 @@ func PrepareResultFieldData(sample []*schemapb.FieldData, topK int64) []*schemap
Data: make([][]byte, 0, topK),
},
}
case *schemapb.ScalarField_GeometryData:
scalar.Scalars.Data = &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: make([][]byte, 0, topK),
},
}
case *schemapb.ScalarField_ArrayData:
scalar.Scalars.Data = &schemapb.ScalarField_ArrayData{
ArrayData: &schemapb.ArrayArray{
@ -781,6 +800,17 @@ func AppendFieldData(dst, src []*schemapb.FieldData, idx int64) (appendSize int6
}
/* #nosec G103 */
appendSize += int64(unsafe.Sizeof(srcScalar.JsonData.Data[idx]))
case *schemapb.ScalarField_GeometryData:
if dstScalar.GetGeometryData() == nil {
dstScalar.Data = &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: [][]byte{srcScalar.GeometryData.Data[idx]},
},
}
} else {
dstScalar.GetGeometryData().Data = append(dstScalar.GetGeometryData().Data, srcScalar.GeometryData.Data[idx])
}
appendSize += int64(unsafe.Sizeof(srcScalar.GeometryData.Data[idx]))
default:
log.Error("Not supported field type", zap.String("field type", fieldData.Type.String()))
}
@ -899,6 +929,8 @@ func DeleteFieldData(dst []*schemapb.FieldData) {
dstScalar.GetStringData().Data = dstScalar.GetStringData().Data[:len(dstScalar.GetStringData().Data)-1]
case *schemapb.ScalarField_JsonData:
dstScalar.GetJsonData().Data = dstScalar.GetJsonData().Data[:len(dstScalar.GetJsonData().Data)-1]
case *schemapb.ScalarField_GeometryData:
dstScalar.GetGeometryData().Data = dstScalar.GetGeometryData().Data[:len(dstScalar.GetGeometryData().Data)-1]
default:
log.Error("wrong field type added", zap.String("field type", fieldData.Type.String()))
}
@ -1036,6 +1068,16 @@ func MergeFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData) error
} else {
dstScalar.GetJsonData().Data = append(dstScalar.GetJsonData().Data, srcScalar.JsonData.Data...)
}
case *schemapb.ScalarField_GeometryData:
if dstScalar.GetGeometryData() == nil {
dstScalar.Data = &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: srcScalar.GeometryData.Data,
},
}
} else {
dstScalar.GetGeometryData().Data = append(dstScalar.GetGeometryData().Data, srcScalar.GeometryData.Data...)
}
case *schemapb.ScalarField_BytesData:
if dstScalar.GetBytesData() == nil {
dstScalar.Data = &schemapb.ScalarField_BytesData{

54
pkg/util/typeutil/schema_test.go
View File

@ -165,6 +165,12 @@ func TestSchema(t *testing.T) {
},
},
},
{
FieldID: 113,
Name: "field_geometry",
IsPrimaryKey: false,
DataType: schemapb.DataType_Geometry,
},
// Do not test on sparse float vector field.
},
}
@ -990,6 +996,21 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
},
FieldId: fieldID,
}
case schemapb.DataType_Geometry:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Geometry,
FieldName: fieldName,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_GeometryData{
GeometryData: &schemapb.GeometryArray{
Data: fieldValue.([][]byte),
},
},
},
},
FieldId: fieldID,
}
default:
log.Error("not supported field type", zap.String("field type", fieldType.String()))
}
@ -1011,6 +1032,7 @@ func TestAppendFieldData(t *testing.T) {
BFloat16VectorFieldName = "BFloat16VectorField"
ArrayFieldName = "ArrayField"
SparseFloatVectorFieldName = "SparseFloatVectorField"
GeometryFieldName = "GeometryField"
BoolFieldID = common.StartOfUserFieldID + 1
Int32FieldID = common.StartOfUserFieldID + 2
Int64FieldID = common.StartOfUserFieldID + 3
@ -1022,6 +1044,7 @@ func TestAppendFieldData(t *testing.T) {
BFloat16VectorFieldID = common.StartOfUserFieldID + 9
ArrayFieldID = common.StartOfUserFieldID + 10
SparseFloatVectorFieldID = common.StartOfUserFieldID + 11
GeometryFieldID = common.StartOfUserFieldID + 12
)
BoolArray := []bool{true, false}
Int32Array := []int32{1, 2}
@ -1061,8 +1084,13 @@ func TestAppendFieldData(t *testing.T) {
CreateSparseFloatRow([]uint32{60, 80, 230}, []float32{2.1, 2.2, 2.3}),
},
}
// POINT (30.123 -10.456) and LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)
GeometryArray := [][]byte{
{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40},
{0x01, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40, 0x03, 0xA6, 0xB4, 0xA6, 0xA4, 0xD2, 0xC5, 0xC0, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A},
}
result := make([]*schemapb.FieldData, 11)
result := make([]*schemapb.FieldData, 12)
var fieldDataArray1 []*schemapb.FieldData
fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1))
@ -1075,6 +1103,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(BFloat16VectorFieldName, BFloat16VectorFieldID, schemapb.DataType_BFloat16Vector, BFloat16Vector[0:Dim*2], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(ArrayFieldName, ArrayFieldID, schemapb.DataType_Array, ArrayArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(SparseFloatVectorFieldName, SparseFloatVectorFieldID, schemapb.DataType_SparseFloatVector, SparseFloatVector.Contents[0], SparseFloatVector.Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(GeometryFieldName, GeometryFieldID, schemapb.DataType_Geometry, GeometryArray[0:1], 1))
var fieldDataArray2 []*schemapb.FieldData
fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1))
@ -1088,6 +1117,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(BFloat16VectorFieldName, BFloat16VectorFieldID, schemapb.DataType_BFloat16Vector, BFloat16Vector[2*Dim:4*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(ArrayFieldName, ArrayFieldID, schemapb.DataType_Array, ArrayArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(SparseFloatVectorFieldName, SparseFloatVectorFieldID, schemapb.DataType_SparseFloatVector, SparseFloatVector.Contents[1], SparseFloatVector.Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(GeometryFieldName, GeometryFieldID, schemapb.DataType_Geometry, GeometryArray[1:2], 1))
AppendFieldData(result, fieldDataArray1, 0)
AppendFieldData(result, fieldDataArray2, 0)
@ -1103,6 +1133,7 @@ func TestAppendFieldData(t *testing.T) {
assert.Equal(t, BFloat16Vector, result[8].GetVectors().Data.(*schemapb.VectorField_Bfloat16Vector).Bfloat16Vector)
assert.Equal(t, ArrayArray, result[9].GetScalars().GetArrayData().Data)
assert.Equal(t, SparseFloatVector, result[10].GetVectors().GetSparseFloatVector())
assert.Equal(t, GeometryArray, result[11].GetScalars().GetGeometryData().Data)
}
func TestDeleteFieldData(t *testing.T) {
@ -1114,6 +1145,7 @@ func TestDeleteFieldData(t *testing.T) {
FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField"
JSONFieldName = "JSONField"
GeometryFieldName = "GeometryField"
BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField"
Float16VectorFieldName = "Float16VectorField"
@ -1128,6 +1160,7 @@ func TestDeleteFieldData(t *testing.T) {
FloatFieldID
DoubleFieldID
JSONFieldID
GeometryFiledID
BinaryVectorFieldID
FloatVectorFieldID
Float16VectorFieldID
@ -1140,6 +1173,11 @@ func TestDeleteFieldData(t *testing.T) {
FloatArray := []float32{1.0, 2.0}
DoubleArray := []float64{11.0, 22.0}
JSONArray := [][]byte{[]byte("{\"hello\":0}"), []byte("{\"key\":1}")}
// POINT (30.123 -10.456) and LINESTRING (30.123 -10.456, 10.789 30.123, -40.567 40.890)
GeometryArray := [][]byte{
{0x01, 0x01, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40},
{0x01, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A, 0x0D, 0x1B, 0x4F, 0x4F, 0x9A, 0x3D, 0x40, 0x03, 0xA6, 0xB4, 0xA6, 0xA4, 0xD2, 0xC5, 0xC0, 0xD2, 0x4A, 0x4D, 0x6A, 0x8B, 0x3C, 0x5C, 0x0A},
}
BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{
@ -1167,6 +1205,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatFieldName, FloatFieldID, schemapb.DataType_Float, FloatArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(GeometryFieldName, GeometryFiledID, schemapb.DataType_Geometry, GeometryArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[0:2*Dim], Dim))
@ -1180,6 +1219,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatFieldName, FloatFieldID, schemapb.DataType_Float, FloatArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(GeometryFieldName, GeometryFiledID, schemapb.DataType_Geometry, GeometryArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[2*Dim:4*Dim], Dim))
@ -1195,6 +1235,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, FloatArray[0:1], result1[FloatFieldID-common.StartOfUserFieldID].GetScalars().GetFloatData().Data)
assert.Equal(t, DoubleArray[0:1], result1[DoubleFieldID-common.StartOfUserFieldID].GetScalars().GetDoubleData().Data)
assert.Equal(t, JSONArray[0:1], result1[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, GeometryArray[0:1], result1[GeometryFiledID-common.StartOfUserFieldID].GetScalars().GetGeometryData().Data)
assert.Equal(t, BinaryVector[0:Dim/8], result1[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[0:Dim], result1[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[0:2*Dim], result1[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
@ -1212,6 +1253,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, FloatArray[1:2], result2[FloatFieldID-common.StartOfUserFieldID].GetScalars().GetFloatData().Data)
assert.Equal(t, DoubleArray[1:2], result2[DoubleFieldID-common.StartOfUserFieldID].GetScalars().GetDoubleData().Data)
assert.Equal(t, JSONArray[1:2], result2[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, GeometryArray[1:2], result2[GeometryFiledID-common.StartOfUserFieldID].GetScalars().GetGeometryData().Data)
assert.Equal(t, BinaryVector[Dim/8:2*Dim/8], result2[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[Dim:2*Dim], result2[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[2*Dim:4*Dim], result2[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
@ -1572,6 +1614,11 @@ func TestCalcColumnSize(t *testing.T) {
Name: "field_json",
DataType: schemapb.DataType_JSON,
},
{
FieldID: 111,
Name: "field_geometry",
DataType: schemapb.DataType_Geometry,
},
},
}
@ -1597,6 +1644,11 @@ func TestCalcColumnSize(t *testing.T) {
expected += len(v)
}
case schemapb.DataType_Geometry:
data := values.([][]byte)
for _, v := range data {
expected += len(v)
}
default:
expected = binary.Size(fieldValues[field.GetFieldID()])
}

6
tests/integration/compaction/mix_compaction_test.go
View File

@ -52,11 +52,12 @@ func (s *CompactionSuite) TestMixCompaction() {
indexType = integration.IndexFaissIvfFlat
metricType = metric.L2
vecType = schemapb.DataType_FloatVector
testType = schemapb.DataType_Geometry
)
collectionName := "TestCompaction_" + funcutil.GenRandomStr()
schema := integration.ConstructSchemaOfVecDataType(collectionName, dim, true, vecType)
schema := integration.ConstructSchemaOfVecDataType(collectionName, dim, true, vecType, testType)
marshaledSchema, err := proto.Marshal(schema)
s.NoError(err)
@ -92,11 +93,12 @@ func (s *CompactionSuite) TestMixCompaction() {
for i := 0; i < rowNum/batch; i++ {
// insert
fVecColumn := integration.NewFloatVectorFieldData(integration.FloatVecField, batch, dim)
geoColumn := integration.NewGeometryFieldData(integration.GeometryField, batch)
hashKeys := integration.GenerateHashKeys(batch)
insertResult, err := c.Proxy.Insert(ctx, &milvuspb.InsertRequest{
DbName: dbName,
CollectionName: collectionName,
FieldsData: []*schemapb.FieldData{fVecColumn},
FieldsData: []*schemapb.FieldData{fVecColumn, geoColumn},
HashKeys: hashKeys,
NumRows: uint32(batch),
})

26
tests/integration/import/import_test.go
View File

@ -55,6 +55,8 @@ type BulkInsertSuite struct {
vecType schemapb.DataType
indexType indexparamcheck.IndexType
metricType metric.MetricType
expr string
testType schemapb.DataType
}
func (s *BulkInsertSuite) SetupTest() {
@ -68,6 +70,8 @@ func (s *BulkInsertSuite) SetupTest() {
s.vecType = schemapb.DataType_FloatVector
s.indexType = "HNSW"
s.metricType = metric.L2
s.expr = ""
s.testType = schemapb.DataType_None
}
func (s *BulkInsertSuite) run() {
@ -87,9 +91,19 @@ func (s *BulkInsertSuite) run() {
fieldSchema3 := &schemapb.FieldSchema{FieldID: 102, Name: "embeddings", DataType: s.vecType, TypeParams: []*commonpb.KeyValuePair{{Key: common.DimKey, Value: "128"}}}
fieldSchema4 := &schemapb.FieldSchema{FieldID: 103, Name: "embeddings", DataType: s.vecType, TypeParams: []*commonpb.KeyValuePair{}}
if s.vecType != schemapb.DataType_SparseFloatVector {
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema3)
if s.testType == schemapb.DataType_None {
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema3)
} else {
fieldSchema5 := &schemapb.FieldSchema{FieldID: 104, Name: "testField" + schemapb.DataType_name[int32(s.testType)], DataType: s.testType, TypeParams: []*commonpb.KeyValuePair{}}
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema3, fieldSchema5)
}
} else {
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema4)
if s.testType == schemapb.DataType_None {
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema4)
} else {
fieldSchema5 := &schemapb.FieldSchema{FieldID: 104, Name: "testField" + schemapb.DataType_name[int32(s.testType)], DataType: s.testType, TypeParams: []*commonpb.KeyValuePair{}}
schema = integration.ConstructSchema(collectionName, dim, s.autoID, fieldSchema1, fieldSchema2, fieldSchema4, fieldSchema5)
}
}
marshaledSchema, err := proto.Marshal(schema)
@ -203,7 +217,7 @@ func (s *BulkInsertSuite) run() {
s.WaitForLoad(ctx, collectionName)
// search
expr := ""
expr := s.expr
nq := 10
topk := 10
roundDecimal := -1
@ -218,6 +232,12 @@ func (s *BulkInsertSuite) run() {
// s.Equal(nq*topk, len(searchResult.GetResults().GetScores()))
}
func (s *BulkInsertSuite) TestGeometryTypes() {
s.testType = schemapb.DataType_Geometry
s.expr = "st_equals(" + "testField" + schemapb.DataType_name[int32(s.testType)] + ",'POINT (-84.036 39.997)')"
s.run()
}
func (s *BulkInsertSuite) TestMultiFileTypes() {
fileTypeArr := []importutilv2.FileType{importutilv2.JSON, importutilv2.Numpy, importutilv2.Parquet, importutilv2.CSV}

4
tests/integration/util_insert.go
View File

@ -135,6 +135,10 @@ func NewStringFieldData(fieldName string, numRows int) *schemapb.FieldData {
return testutils.NewStringFieldData(fieldName, numRows)
}
func NewGeometryFieldData(fieldName string, numRows int) *schemapb.FieldData {
return testutils.NewGeometryFieldDataWktFormat(fieldName, numRows)
}
func NewFloatVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return testutils.NewFloatVectorFieldData(fieldName, numRows, dim)
}

57
tests/integration/util_schema.go
View File

@ -34,6 +34,7 @@ const (
DoubleField = "doubleField"
VarCharField = "varCharField"
JSONField = "jsonField"
GeometryField = "geometryField"
FloatVecField = "floatVecField"
BinVecField = "binVecField"
Float16VecField = "float16VecField"
@ -83,7 +84,7 @@ func ConstructSchema(collection string, dim int, autoID bool, fields ...*schemap
}
}
func ConstructSchemaOfVecDataType(collection string, dim int, autoID bool, dataType schemapb.DataType) *schemapb.CollectionSchema {
func ConstructSchemaOfVecDataType(collection string, dim int, autoID bool, dataType ...schemapb.DataType) *schemapb.CollectionSchema {
pk := &schemapb.FieldSchema{
FieldID: 100,
Name: Int64Field,
@ -96,33 +97,41 @@ func ConstructSchemaOfVecDataType(collection string, dim int, autoID bool, dataT
}
var name string
var typeParams []*commonpb.KeyValuePair
switch dataType {
case schemapb.DataType_FloatVector:
name = FloatVecField
typeParams = []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: fmt.Sprintf("%d", dim),
},
var fieldSchemaArray []*schemapb.FieldSchema
fieldSchemaArray = append(fieldSchemaArray, pk)
for i := 0; i < len(dataType); i++ {
switch dataType[i] {
case schemapb.DataType_FloatVector:
name = FloatVecField
typeParams = []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: fmt.Sprintf("%d", dim),
},
}
case schemapb.DataType_SparseFloatVector:
name = SparseFloatVecField
typeParams = nil
case schemapb.DataType_Geometry:
name = GeometryField
typeParams = nil
default:
panic("unsupported data type")
}
case schemapb.DataType_SparseFloatVector:
name = SparseFloatVecField
typeParams = nil
default:
panic("unsupported data type")
}
fVec := &schemapb.FieldSchema{
FieldID: 101,
Name: name,
IsPrimaryKey: false,
Description: "",
DataType: dataType,
TypeParams: typeParams,
IndexParams: nil,
sche := &schemapb.FieldSchema{
FieldID: 101 + int64(i),
Name: name,
IsPrimaryKey: false,
Description: "",
DataType: dataType[i],
TypeParams: typeParams,
IndexParams: nil,
}
fieldSchemaArray = append(fieldSchemaArray, sche)
}
return &schemapb.CollectionSchema{
Name: collection,
AutoID: autoID,
Fields: []*schemapb.FieldSchema{pk, fVec},
Fields: fieldSchemaArray,
}
}