Merge pull request #3901 from influxdata/dom/rustls

feat: rustls as TLS backend
pull/24376/head
kodiakhq[bot] 2022-03-03 12:11:01 +00:00 committed by GitHub
commit 208c3673ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 275 additions and 172 deletions

View File

@ -38,8 +38,9 @@ workspace-members = [
"server_benchmarks",
]
third-party = [
{ name = "azure_core", git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "cb5aba657fe378009c38e979434b2bc93ac0a3d5" },
{ name = "azure_storage", git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "cb5aba657fe378009c38e979434b2bc93ac0a3d5" },
{ name = "azure_core", git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244" },
{ name = "azure_storage", git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244" },
{ name = "azure_storage_blobs", git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244" },
{ name = "cloud-storage" },
{ name = "criterion" },
{ name = "pprof" },

342
Cargo.lock generated
View File

@ -229,10 +229,11 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "azure_core"
version = "0.1.0"
source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=cb5aba657fe378009c38e979434b2bc93ac0a3d5#cb5aba657fe378009c38e979434b2bc93ac0a3d5"
version = "0.1.1"
source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=15de10cb58cf5cb271334a20e35fa9fbedd10244#15de10cb58cf5cb271334a20e35fa9fbedd10244"
dependencies = [
"async-trait",
"base64 0.13.0",
"bytes",
"chrono",
"dyn-clone",
@ -241,6 +242,7 @@ dependencies = [
"http",
"log",
"oauth2",
"pin-project",
"rand",
"reqwest",
"rustc_version",
@ -255,10 +257,37 @@ dependencies = [
[[package]]
name = "azure_storage"
version = "0.1.0"
source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=cb5aba657fe378009c38e979434b2bc93ac0a3d5#cb5aba657fe378009c38e979434b2bc93ac0a3d5"
source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=15de10cb58cf5cb271334a20e35fa9fbedd10244#15de10cb58cf5cb271334a20e35fa9fbedd10244"
dependencies = [
"RustyXML",
"async-trait",
"azure_core",
"base64 0.13.0",
"bytes",
"chrono",
"futures",
"hmac 0.12.1",
"http",
"log",
"once_cell",
"serde",
"serde-xml-rs",
"serde_derive",
"serde_json",
"sha2 0.10.2",
"thiserror",
"url",
"uuid",
]
[[package]]
name = "azure_storage_blobs"
version = "0.1.0"
source = "git+https://github.com/Azure/azure-sdk-for-rust.git?rev=15de10cb58cf5cb271334a20e35fa9fbedd10244#15de10cb58cf5cb271334a20e35fa9fbedd10244"
dependencies = [
"RustyXML",
"azure_core",
"azure_storage",
"base64 0.13.0",
"bytes",
"chrono",
@ -266,8 +295,6 @@ dependencies = [
"http",
"log",
"md5",
"once_cell",
"ring",
"serde",
"serde-xml-rs",
"serde_derive",
@ -375,6 +402,15 @@ dependencies = [
"generic-array 0.14.5",
]
[[package]]
name = "block-buffer"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324"
dependencies = [
"generic-array 0.14.5",
]
[[package]]
name = "block-padding"
version = "0.1.5"
@ -610,9 +646,10 @@ dependencies = [
"hex",
"jsonwebtoken",
"lazy_static",
"openssl",
"pem 0.8.3",
"percent-encoding",
"reqwest",
"ring",
"serde",
"serde_json",
"tokio",
@ -847,6 +884,16 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "crypto-common"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
dependencies = [
"generic-array 0.14.5",
"typenum",
]
[[package]]
name = "crypto-mac"
version = "0.11.1"
@ -879,6 +926,15 @@ dependencies = [
"memchr",
]
[[package]]
name = "ct-logs"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
dependencies = [
"sct 0.6.1",
]
[[package]]
name = "ctor"
version = "0.1.21"
@ -1124,6 +1180,17 @@ dependencies = [
"generic-array 0.14.5",
]
[[package]]
name = "digest"
version = "0.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506"
dependencies = [
"block-buffer 0.10.2",
"crypto-common",
"subtle",
]
[[package]]
name = "dirs"
version = "4.0.0"
@ -1361,21 +1428,6 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.0.1"
@ -1731,6 +1783,15 @@ dependencies = [
"digest 0.9.0",
]
[[package]]
name = "hmac"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
dependencies = [
"digest 0.10.3",
]
[[package]]
name = "http"
version = "0.2.6"
@ -1801,6 +1862,23 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [
"ct-logs",
"futures-util",
"hyper",
"log",
"rustls 0.19.1",
"rustls-native-certs 0.5.0",
"tokio",
"tokio-rustls 0.22.0",
"webpki 0.21.4",
]
[[package]]
name = "hyper-rustls"
version = "0.23.0"
@ -1809,9 +1887,12 @@ checksum = "d87c48c02e0dc5e3b849a2041db3029fd066650f8f717c07bf8ed78ccb895cac"
dependencies = [
"http",
"hyper",
"rustls",
"log",
"rustls 0.20.4",
"rustls-native-certs 0.6.1",
"tokio",
"tokio-rustls",
"tokio-rustls 0.23.2",
"webpki-roots 0.22.2",
]
[[package]]
@ -1826,19 +1907,6 @@ dependencies = [
"tokio-io-timeout",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes",
"hyper",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]]
name = "ident_case"
version = "1.0.1"
@ -2408,21 +2476,21 @@ dependencies = [
"http",
"http-body",
"hyper",
"hyper-rustls 0.23.0",
"hyper-timeout",
"hyper-tls",
"jsonpath_lib",
"k8s-openapi",
"kube-core",
"openssl",
"pem 1.0.2",
"pin-project",
"rustls 0.20.4",
"rustls-pemfile",
"secrecy",
"serde",
"serde_json",
"serde_yaml",
"thiserror",
"tokio",
"tokio-native-tls",
"tokio-util 0.6.9",
"tower",
"tower-http",
@ -2906,24 +2974,6 @@ dependencies = [
"workspace-hack",
]
[[package]]
name = "native-tls"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "nibble_vec"
version = "0.1.0"
@ -3110,7 +3160,7 @@ dependencies = [
"serde",
"serde_json",
"serde_path_to_error",
"sha2",
"sha2 0.9.9",
"thiserror",
"url",
]
@ -3131,6 +3181,7 @@ dependencies = [
"async-trait",
"azure_core",
"azure_storage",
"azure_storage_blobs",
"bytes",
"chrono",
"cloud-storage",
@ -3138,7 +3189,7 @@ dependencies = [
"futures",
"futures-test",
"hyper",
"hyper-tls",
"hyper-rustls 0.23.0",
"indexmap",
"itertools",
"observability_deps",
@ -3190,39 +3241,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
dependencies = [
"bitflags",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-sys",
]
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "openssl-sys"
version = "0.9.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
dependencies = [
"autocfg",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "ordered-float"
version = "1.1.1"
@ -3615,12 +3639,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
[[package]]
name = "plotters"
version = "0.3.1"
@ -4143,30 +4161,27 @@ dependencies = [
"http",
"http-body",
"hyper",
"hyper-rustls",
"hyper-tls",
"hyper-rustls 0.23.0",
"ipnet",
"js-sys",
"lazy_static",
"log",
"mime",
"native-tls",
"percent-encoding",
"pin-project-lite",
"rustls",
"rustls 0.20.4",
"rustls-pemfile",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-rustls 0.23.2",
"tokio-util 0.6.9",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
"webpki-roots 0.22.2",
"winreg",
]
@ -4292,7 +4307,7 @@ dependencies = [
"futures",
"http",
"hyper",
"hyper-tls",
"hyper-rustls 0.22.1",
"lazy_static",
"log",
"rusoto_credential",
@ -4347,7 +4362,7 @@ dependencies = [
"digest 0.9.0",
"futures",
"hex",
"hmac",
"hmac 0.11.0",
"http",
"hyper",
"log",
@ -4357,7 +4372,7 @@ dependencies = [
"rusoto_credential",
"rustc_version",
"serde",
"sha2",
"sha2 0.9.9",
"tokio",
]
@ -4396,6 +4411,19 @@ dependencies = [
"winapi",
]
[[package]]
name = "rustls"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
dependencies = [
"base64 0.13.0",
"log",
"ring",
"sct 0.6.1",
"webpki 0.21.4",
]
[[package]]
name = "rustls"
version = "0.20.4"
@ -4404,8 +4432,32 @@ checksum = "4fbfeb8d0ddb84706bc597a5574ab8912817c52a397f819e5b614e2265206921"
dependencies = [
"log",
"ring",
"sct",
"webpki",
"sct 0.7.0",
"webpki 0.22.0",
]
[[package]]
name = "rustls-native-certs"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
"openssl-probe",
"rustls 0.19.1",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]]
@ -4519,6 +4571,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "sct"
version = "0.7.0"
@ -4589,9 +4651,9 @@ dependencies = [
[[package]]
name = "serde-xml-rs"
version = "0.4.1"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0bf1ba0696ccf0872866277143ff1fd14d22eec235d2b23702f95e6660f7dfa"
checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa"
dependencies = [
"log",
"serde",
@ -4784,6 +4846,17 @@ dependencies = [
"opaque-debug 0.3.0",
]
[[package]]
name = "sha2"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676"
dependencies = [
"cfg-if",
"cpufeatures",
"digest 0.10.3",
]
[[package]]
name = "sharded-slab"
version = "0.1.4"
@ -4948,7 +5021,7 @@ dependencies = [
"futures-util",
"hashlink",
"hex",
"hmac",
"hmac 0.11.0",
"indexmap",
"itoa 1.0.1",
"libc",
@ -4959,10 +5032,11 @@ dependencies = [
"paste",
"percent-encoding",
"rand",
"rustls 0.19.1",
"serde",
"serde_json",
"sha-1 0.9.8",
"sha2",
"sha2 0.9.9",
"smallvec",
"sqlformat",
"sqlx-rt",
@ -4971,6 +5045,8 @@ dependencies = [
"tokio-stream",
"url",
"uuid",
"webpki 0.21.4",
"webpki-roots 0.21.1",
"whoami",
]
@ -5001,7 +5077,7 @@ dependencies = [
"proc-macro2",
"quote",
"serde_json",
"sha2",
"sha2 0.9.9",
"sqlx-core",
"sqlx-rt",
"syn",
@ -5014,10 +5090,9 @@ version = "0.5.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b555e70fbbf84e269ec3858b7a6515bcfe7a166a7cc9c636dd6efd20431678b6"
dependencies = [
"native-tls",
"once_cell",
"tokio",
"tokio-native-tls",
"tokio-rustls 0.22.0",
]
[[package]]
@ -5357,13 +5432,14 @@ dependencies = [
]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
name = "tokio-rustls"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
"native-tls",
"rustls 0.19.1",
"tokio",
"webpki 0.21.4",
]
[[package]]
@ -5372,9 +5448,9 @@ version = "0.23.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b"
dependencies = [
"rustls",
"rustls 0.20.4",
"tokio",
"webpki",
"webpki 0.22.0",
]
[[package]]
@ -5820,12 +5896,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "vec_map"
version = "0.8.2"
@ -5950,6 +6020,16 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "webpki"
version = "0.22.0"
@ -5960,13 +6040,22 @@ dependencies = [
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940"
dependencies = [
"webpki 0.21.4",
]
[[package]]
name = "webpki-roots"
version = "0.22.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449"
dependencies = [
"webpki",
"webpki 0.22.0",
]
[[package]]
@ -6139,6 +6228,7 @@ dependencies = [
"hashbrown 0.11.2",
"hashbrown 0.12.0",
"hyper",
"hyper-rustls 0.23.0",
"indexmap",
"libc",
"log",
@ -6155,16 +6245,18 @@ dependencies = [
"regex-automata",
"regex-syntax",
"reqwest",
"rustls 0.20.4",
"scopeguard",
"serde",
"serde_json",
"sha2",
"sha2 0.9.9",
"smallvec",
"sqlx",
"sqlx-core",
"sqlx-macros",
"syn",
"tokio",
"tokio-rustls 0.23.2",
"tokio-stream",
"tokio-util 0.6.9",
"tokio-util 0.7.0",

View File

@ -65,7 +65,6 @@ To compile and run InfluxDB IOx from source, you'll need the following:
* [Clang](#clang)
* [lld (on Linux)](#lld)
* [protoc (on Apple Silicon)](#protoc)
* [openssl](#openssl)
#### Rust
@ -137,20 +136,6 @@ PROTOC_INCLUDE=/opt/homebrew/include
IOx should then build correctly.
#### openssl
Install the `openssl` library.
Check if you have `openssl` installed with `$ openssl version`. If it's missing then openssl can typically be installed with your system package manager.
More information can be found [here](https://github.com/sfackler/rust-openssl/issues/1021) on build errors due to "openssl" not being installed.
If you don't install `openssl`, build would fail with an error like:
```shell
error: failed to run custom build command for `openssl-sys v0.9.72`
```
### Clone the repository
Clone this repository using `git`.

View File

@ -10,6 +10,6 @@ influxdb_line_protocol = { path = "../influxdb_line_protocol" }
mutable_batch = { path = "../mutable_batch" }
predicate = { path = "../predicate" }
schema = { path = "../schema" }
sqlx = { version = "0.5", features = [ "runtime-tokio-native-tls" , "postgres", "uuid" ] }
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "postgres", "uuid"] }
uuid = { version = "0.8", features = ["v4"] }
workspace-hack = { path = "../workspace-hack"}

View File

@ -7,7 +7,7 @@ edition = "2021"
[dependencies] # In alphabetical order
bytes = "1.0"
futures = { version = "0.3", default-features = false }
reqwest = { version = "0.11", features = ["stream", "json"] }
reqwest = { version = "0.11", default-features = false, features = ["stream", "json", "rustls-tls"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0.79"
snafu = "0.7"

View File

@ -116,7 +116,7 @@ hex = "0.4.2"
predicates = "2.1.0"
rand = "0.8.3"
regex = "1"
reqwest = { version = "0.11", features = ["json"] }
reqwest = { version = "0.11", default-features = false, features = ["json", "rustls-tls"] }
tempfile = "3.1.0"
[features]

View File

@ -13,7 +13,7 @@ metric = { version = "0.1.0", path = "../metric" }
mutable_batch = { path = "../mutable_batch" }
observability_deps = { path = "../observability_deps" }
snafu = "0.7"
sqlx = { version = "0.5", features = [ "runtime-tokio-native-tls" , "postgres", "uuid" ] }
sqlx = { version = "0.5", features = [ "runtime-tokio-rustls" , "postgres", "uuid" ] }
sqlx-hotswap-pool = { path = "../sqlx-hotswap-pool" }
time = { version = "0.1.0", path = "../time" }
tokio = { version = "1.17", features = ["io-util", "macros", "parking_lot", "rt-multi-thread", "time"] }

View File

@ -20,7 +20,7 @@ clap = { version = "3", features = ["derive", "env"] }
dotenv = "0.15"
futures = "0.3"
k8s-openapi = { version = "0.14", features = ["v1_19", "schemars"], default-features = false }
kube = { version = "0.69", features = ["derive"] }
kube = { version = "0.69", default-features = false, features = ["client", "rustls-tls", "derive"] }
kube-derive = { version = "0.69", default-features = false } # only needed to opt out of schema
kube-runtime = "0.69"
prost = "0.9"

View File

@ -8,40 +8,41 @@ edition = "2021"
async-trait = "0.1.42"
# Microsoft Azure Blob storage integration
# In order to support tokio 1.0 and delimiters, needed to pull in unreleased azure sdk
azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "cb5aba657fe378009c38e979434b2bc93ac0a3d5", optional = true }
azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "cb5aba657fe378009c38e979434b2bc93ac0a3d5", optional = true, default-features = false, features = ["table", "blob", "queue"] }
azure_core = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244", optional = true, default-features = false, features = ["enable_reqwest_rustls"] }
azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244", optional = true, default-features = false, features = ["account"] }
azure_storage_blobs = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev = "15de10cb58cf5cb271334a20e35fa9fbedd10244", optional = true, default-features = false, features = ["enable_reqwest_rustls"] }
bytes = "1.0"
chrono = { version = "0.4", default-features = false, features = ["clock"] }
# Google Cloud Storage integration
cloud-storage = {version = "0.11.0", optional = true}
cloud-storage = {version = "0.11.0", optional = true, default-features = false, features = ["rustls-tls"] }
futures = "0.3"
# for rusoto
hyper = { version = "0.14", optional = true, default-features = false }
# for rusoto
hyper-tls = { version = "0.5.0", optional = true, default-features = false }
hyper-rustls = { version = "0.23.0", optional = true, default-features = false, features = ["webpki-tokio", "http1", "http2", "tls12"] }
indexmap = { version = "1.7", optional = true, features = ["std"] }
itertools = "0.10.1"
observability_deps = { path = "../observability_deps" }
percent-encoding = "2.1"
# rusoto crates are for Amazon S3 integration
rusoto_core = { version = "0.47.0", optional = true}
rusoto_credential = { version = "0.47.0", optional = true}
rusoto_s3 = { version = "0.47.0", optional = true}
rusoto_core = { version = "0.47.0", optional = true, default-features = false, features = ["rustls"] }
rusoto_credential = { version = "0.47.0", optional = true, default-features = false }
rusoto_s3 = { version = "0.47.0", optional = true, default-features = false, features = ["rustls"] }
snafu = "0.7"
tokio = { version = "1.17", features = ["fs", "io-util", "macros", "parking_lot", "rt-multi-thread", "time"] }
# Filesystem integration
tokio-util = { version = "0.7.0", features = [ "codec", "io" ] }
reqwest = { version = "0.11", optional = true }
reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls"] }
# Filesystem integration
walkdir = "2"
tempfile = "3.1.0"
workspace-hack = { path = "../workspace-hack"}
[features]
azure = ["azure_core", "azure_storage", "indexmap", "reqwest"]
azure_test = ["azure", "azure_core/azurite_workaround", "azure_storage/azurite_workaround"]
azure = ["azure_core", "azure_storage_blobs", "azure_storage", "indexmap", "reqwest"]
azure_test = ["azure", "azure_core/azurite_workaround", "azure_storage/azurite_workaround", "azure_storage_blobs/azurite_workaround"]
gcp = ["cloud-storage"]
aws = ["rusoto_core", "rusoto_credential", "rusoto_s3", "hyper", "hyper-tls"]
aws = ["rusoto_core", "rusoto_credential", "rusoto_s3", "hyper", "hyper-rustls"]
[dev-dependencies] # In alphabetical order
dotenv = "0.15.0"

View File

@ -12,7 +12,6 @@ use futures::{
Future, StreamExt, TryStreamExt,
};
use hyper::client::Builder as HyperBuilder;
use hyper_tls::HttpsConnector;
use observability_deps::tracing::{debug, warn};
use rusoto_core::ByteStream;
use rusoto_credential::{InstanceMetadataProvider, StaticProvider};
@ -393,7 +392,25 @@ pub(crate) fn new_s3(
let mut builder = HyperBuilder::default();
builder.pool_max_idle_per_host(max_connections.get());
let connector = HttpsConnector::new();
// For testing purposes, allow connections to HTTP endpoints.
#[cfg(test)]
let connector = hyper_rustls::HttpsConnectorBuilder::new()
.with_webpki_roots()
.https_or_http()
.enable_http1()
.enable_http2()
.build();
// In production, we should not allow plain-text connections when pushing
// parquet files to object storage, so only TLS connections are allowed.
#[cfg(not(test))]
let connector = hyper_rustls::HttpsConnectorBuilder::new()
.with_webpki_roots()
.https_only()
.enable_http1()
.enable_http2()
.build();
let http_client = rusoto_core::request::HttpClient::from_builder(builder, connector);
let client = match (access_key_id, secret_access_key, session_token) {

View File

@ -5,10 +5,10 @@ use crate::{
GetResult, ListResult, ObjectMeta, ObjectStoreApi, ObjectStorePath,
};
use async_trait::async_trait;
use azure_core::prelude::*;
use azure_storage::{
blob::prelude::{AsBlobClient, AsContainerClient, ContainerClient},
core::clients::{AsStorageClient, StorageAccountClient},
use azure_core::{prelude::*, HttpClient};
use azure_storage::core::prelude::*;
use azure_storage_blobs::{
prelude::{AsBlobClient, AsContainerClient, ContainerClient},
DeleteSnapshotsMethod,
};
use bytes::Bytes;

View File

@ -9,7 +9,7 @@ description = "Workaround for the lack of dyanmic credential update support in s
publish = false
[dependencies]
sqlx = { version = "0.5.11", features = ["runtime-tokio-native-tls", "postgres", "json", "tls"] }
sqlx = { version = "0.5.11", features = ["runtime-tokio-rustls", "postgres", "json", "tls"] }
either = "1.6.1"
tokio = { version = "1.17", features = ["rt-multi-thread", "macros", "parking_lot"] }
tokio-stream = { version = "0.1", default_features = false }

View File

@ -31,6 +31,7 @@ getrandom = { version = "0.2", default-features = false, features = ["std"] }
hashbrown-a6292c17cd707f01 = { package = "hashbrown", version = "0.11", features = ["ahash", "inline-more", "raw"] }
hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["ahash", "inline-more", "raw"] }
hyper = { version = "0.14", features = ["client", "full", "h2", "http1", "http2", "runtime", "server", "socket2", "stream", "tcp"] }
hyper-rustls = { version = "0.23", features = ["http1", "http2", "log", "logging", "native-tokio", "rustls-native-certs", "tls12", "tokio-runtime", "webpki-roots", "webpki-tokio"] }
indexmap = { version = "1", default-features = false, features = ["std"] }
log = { version = "0.4", default-features = false, features = ["std"] }
memchr = { version = "2", features = ["std"] }
@ -45,14 +46,16 @@ rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha
regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] }
regex-automata = { version = "0.1", features = ["regex-syntax", "std"] }
regex-syntax = { version = "0.6", features = ["unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] }
reqwest = { version = "0.11", features = ["__tls", "default-tls", "hyper-tls", "json", "native-tls-crate", "serde_json", "tokio-native-tls"] }
reqwest = { version = "0.11", default-features = false, features = ["__rustls", "__tls", "hyper-rustls", "json", "rustls", "rustls-pemfile", "rustls-tls", "rustls-tls-webpki-roots", "serde_json", "tokio-rustls", "webpki-roots"] }
rustls = { version = "0.20", features = ["dangerous_configuration", "log", "logging", "tls12"] }
serde = { version = "1", features = ["alloc", "derive", "rc", "serde_derive", "std"] }
serde_json = { version = "1", features = ["alloc", "indexmap", "preserve_order", "raw_value", "std"] }
sha2 = { version = "0.9", features = ["std"] }
smallvec = { version = "1", default-features = false, features = ["union"] }
sqlx = { version = "0.5", features = ["_rt-tokio", "json", "macros", "migrate", "postgres", "runtime-tokio-native-tls", "sqlx-macros", "tls", "uuid"] }
sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-native-tls", "base64", "crc", "dirs", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-native-tls", "serde", "serde_json", "sha-1", "sha2", "tokio-stream", "uuid", "whoami"] }
sqlx = { version = "0.5", features = ["_rt-tokio", "json", "macros", "migrate", "postgres", "runtime-tokio-rustls", "sqlx-macros", "tls", "uuid"] }
sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "base64", "crc", "dirs", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-rustls", "rustls", "serde", "serde_json", "sha-1", "sha2", "tokio-stream", "uuid", "webpki", "webpki-roots", "whoami"] }
tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] }
tokio-rustls = { version = "0.23", default-features = false, features = ["logging", "tls12"] }
tokio-stream = { version = "0.1", features = ["fs", "net", "time"] }
tokio-util-3b31131e45eafb45 = { package = "tokio-util", version = "0.6", features = ["codec", "io", "slab", "time"] }
tokio-util-ca01ad9e24f5d932 = { package = "tokio-util", version = "0.7", features = ["codec", "io"] }
@ -93,8 +96,8 @@ serde = { version = "1", features = ["alloc", "derive", "rc", "serde_derive", "s
serde_json = { version = "1", features = ["alloc", "indexmap", "preserve_order", "raw_value", "std"] }
sha2 = { version = "0.9", features = ["std"] }
smallvec = { version = "1", default-features = false, features = ["union"] }
sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-native-tls", "base64", "crc", "dirs", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-native-tls", "serde", "serde_json", "sha-1", "sha2", "tokio-stream", "uuid", "whoami"] }
sqlx-macros = { version = "0.5", default-features = false, features = ["_rt-tokio", "json", "migrate", "postgres", "runtime-tokio-native-tls", "serde_json", "sha2", "uuid"] }
sqlx-core = { version = "0.5", default-features = false, features = ["_rt-tokio", "_tls-rustls", "base64", "crc", "dirs", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-rustls", "rustls", "serde", "serde_json", "sha-1", "sha2", "tokio-stream", "uuid", "webpki", "webpki-roots", "whoami"] }
sqlx-macros = { version = "0.5", default-features = false, features = ["_rt-tokio", "json", "migrate", "postgres", "runtime-tokio-rustls", "serde_json", "sha2", "uuid"] }
syn = { version = "1", features = ["clone-impls", "derive", "extra-traits", "full", "parsing", "printing", "proc-macro", "quote", "visit", "visit-mut"] }
tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] }
tokio-stream = { version = "0.1", features = ["fs", "net", "time"] }
@ -102,18 +105,21 @@ uuid = { version = "0.8", features = ["getrandom", "std", "v4"] }
[target.x86_64-unknown-linux-gnu.dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
tokio-rustls = { version = "0.23" }
[target.x86_64-unknown-linux-gnu.build-dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
[target.x86_64-apple-darwin.dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
tokio-rustls = { version = "0.23" }
[target.x86_64-apple-darwin.build-dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
[target.aarch64-apple-darwin.dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
tokio-rustls = { version = "0.23" }
[target.aarch64-apple-darwin.build-dependencies]
libc = { version = "0.2", features = ["extra_traits", "std"] }
@ -121,11 +127,12 @@ libc = { version = "0.2", features = ["extra_traits", "std"] }
[target.x86_64-pc-windows-msvc.dependencies]
scopeguard = { version = "1", features = ["use_std"] }
tokio = { version = "1", default-features = false, features = ["winapi"] }
winapi = { version = "0.3", default-features = false, features = ["basetsd", "cfg", "consoleapi", "errhandlingapi", "evntrace", "fileapi", "handleapi", "impl-debug", "impl-default", "in6addr", "inaddr", "ioapiset", "knownfolders", "libloaderapi", "lmcons", "minschannel", "minwinbase", "minwindef", "mswsock", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "profileapi", "schannel", "securitybaseapi", "shellapi", "shlobj", "sspi", "std", "stringapiset", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "wincrypt", "windef", "winerror", "winioctl", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip"] }
tokio-rustls = { version = "0.23" }
winapi = { version = "0.3", default-features = false, features = ["basetsd", "cfg", "consoleapi", "errhandlingapi", "evntrace", "fileapi", "handleapi", "impl-debug", "impl-default", "in6addr", "inaddr", "ioapiset", "knownfolders", "libloaderapi", "lmcons", "minschannel", "minwinbase", "minwindef", "mswsock", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "profileapi", "schannel", "securitybaseapi", "shellapi", "shlobj", "sspi", "std", "stringapiset", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "wincrypt", "windef", "winerror", "winioctl", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
[target.x86_64-pc-windows-msvc.build-dependencies]
scopeguard = { version = "1", features = ["use_std"] }
tokio = { version = "1", default-features = false, features = ["winapi"] }
winapi = { version = "0.3", default-features = false, features = ["basetsd", "cfg", "consoleapi", "errhandlingapi", "evntrace", "fileapi", "handleapi", "impl-debug", "impl-default", "in6addr", "inaddr", "ioapiset", "knownfolders", "libloaderapi", "lmcons", "minschannel", "minwinbase", "minwindef", "mswsock", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "profileapi", "schannel", "securitybaseapi", "shellapi", "shlobj", "sspi", "std", "stringapiset", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "wincrypt", "windef", "winerror", "winioctl", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip"] }
winapi = { version = "0.3", default-features = false, features = ["basetsd", "cfg", "consoleapi", "errhandlingapi", "evntrace", "fileapi", "handleapi", "impl-debug", "impl-default", "in6addr", "inaddr", "ioapiset", "knownfolders", "libloaderapi", "lmcons", "minschannel", "minwinbase", "minwindef", "mswsock", "namedpipeapi", "ntdef", "ntsecapi", "ntstatus", "objbase", "processenv", "profileapi", "schannel", "securitybaseapi", "shellapi", "shlobj", "sspi", "std", "stringapiset", "synchapi", "sysinfoapi", "timezoneapi", "winbase", "wincon", "wincrypt", "windef", "winerror", "winioctl", "winnt", "winreg", "winsock2", "winuser", "ws2def", "ws2ipdef", "ws2tcpip", "wtypesbase"] }
### END HAKARI SECTION