diff --git a/Cargo.lock b/Cargo.lock index dcda9a15dd..de1e0c2607 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2487,6 +2487,7 @@ dependencies = [ "futures-test", "indexmap", "itertools", + "observability_deps", "percent-encoding", "reqwest", "rusoto_core", diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml index 6803bcf34d..81483e0f14 100644 --- a/object_store/Cargo.toml +++ b/object_store/Cargo.toml @@ -18,6 +18,7 @@ futures = "0.3" # https://github.com/tkaitchuck/aHash/issues/95 indexmap = { version = "~1.6.2", optional = true } itertools = "0.10.1" +observability_deps = { path = "../observability_deps" } percent-encoding = "2.1" # rusoto crates are for Amazon S3 integration rusoto_core = { version = "0.47.0", optional = true} diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs index 4f596eedc2..01841abc54 100644 --- a/object_store/src/aws.rs +++ b/object_store/src/aws.rs @@ -12,6 +12,7 @@ use futures::{ stream::{self, BoxStream}, Future, Stream, StreamExt, TryStreamExt, }; +use observability_deps::tracing::{debug, warn}; use rusoto_core::ByteStream; use rusoto_credential::{InstanceMetadataProvider, StaticProvider}; use rusoto_s3::S3; @@ -474,6 +475,7 @@ impl AmazonS3 { async fn s3_request(future_factory: F) -> Result> where + E: std::error::Error, F: Fn() -> G + Unpin + Clone + Send + Sync + 'static, G: Future>> + Send, H: Future>> + Send, @@ -488,18 +490,25 @@ where match result { Ok(r) => return Ok(r), - Err(e) => { + Err(error) => { attempts += 1; let should_retry = matches!( - e, + error, rusoto_core::RusotoError::Unknown(ref response) if response.status.is_server_error() ); - if attempts > MAX_NUM_RETRIES || !should_retry { - return Err(e); + if attempts > MAX_NUM_RETRIES { + warn!( + ?error, + attempts, "maximum number of retries exceeded for AWS S3 request" + ); + return Err(error); + } else if !should_retry { + return Err(error); } else { + debug!(?error, attempts, "retrying AWS S3 request"); let wait_time = Duration::from_millis(2u64.pow(attempts) * 50); tokio::time::sleep(wait_time).await; }