From f02d846389323276173f3b42fba47329eaffd5c2 Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Thu, 16 Dec 2021 10:10:06 +0100 Subject: [PATCH 1/3] test: `object_store` aws test using localstack This removes 3 "nonexisting region" tests that where testing very specific error behavior that no local emulator (minio and localstack) replicate and that don't add much value. It's better to test our AWS code at all than being to picky. --- .circleci/config.yml | 16 +++++- object_store/src/aws.rs | 112 ---------------------------------------- 2 files changed, 15 insertions(+), 113 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c04ebe4adc..123d360d15 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -170,6 +170,7 @@ jobs: test: docker: - image: quay.io/influxdb/rust:ci + - image: localstack/localstack resource_class: xlarge # use of a smaller executor tends crashes on link environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. @@ -182,18 +183,31 @@ jobs: TEST_INTEGRATION: 1 INFLUXDB_IOX_INTEGRATION_LOCAL: 1 KAFKA_CONNECT: "localhost:9092" + AWS_DEFAULT_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: test + AWS_SECRET_ACCESS_KEY: test + AWS_ENDPOINT: http://127.0.0.1:4566 + INFLUXDB_IOX_BUCKET: iox-test steps: - run: name: Run Kafka # Sudo needed because data directory owned by root but container runs as unprivileged user command: sudo /opt/redpanda/bin/redpanda --redpanda-cfg /etc/redpanda/redpanda.yaml --overprovisioned --smp 1 --memory 1G --reserve-memory 0M background: true + - run: + name: Setup localstack (AWS emulation) + command: | + cd /tmp + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + aws --endpoint-url=http://localhost:4566 s3 mb s3://iox-test - checkout - rust_components - cache_restore - run: name: Cargo test - command: cargo test --workspace --features=kafka + command: cargo test --workspace --features=aws,kafka - cache_save # end to end tests with Heappy (heap profiling enabled) diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs index b1a06a697e..ba19c5183b 100644 --- a/object_store/src/aws.rs +++ b/object_store/src/aws.rs @@ -743,40 +743,6 @@ mod tests { check_credentials(list_with_delimiter(&integration).await).unwrap(); } - #[tokio::test] - async fn s3_test_get_nonexistent_region() { - let mut config = maybe_skip_integration!(); - // Assumes environment variables do not provide credentials to AWS US West 1 - config.region = "us-west-1".into(); - - let integration = ObjectStore::new_amazon_s3( - Some(config.access_key_id), - Some(config.secret_access_key), - config.region, - &config.bucket, - config.endpoint, - config.token, - NonZeroUsize::new(16).unwrap(), - ) - .expect("Valid S3 config"); - - let mut location = integration.new_path(); - location.set_file_name(NON_EXISTENT_NAME); - - let err = get_nonexistent_object(&integration, Some(location)) - .await - .unwrap_err(); - if let Some(ObjectStoreError::AwsObjectStoreError { - source: Error::UnableToListData { source, bucket }, - }) = err.downcast_ref::() - { - assert!(matches!(source, rusoto_core::RusotoError::Unknown(_))); - assert_eq!(bucket, &config.bucket); - } else { - panic!("unexpected error type: {:?}", err); - } - } - #[tokio::test] async fn s3_test_get_nonexistent_location() { let config = maybe_skip_integration!(); @@ -853,46 +819,6 @@ mod tests { } } - #[tokio::test] - async fn s3_test_put_nonexistent_region() { - let mut config = maybe_skip_integration!(); - // Assumes environment variables do not provide credentials to AWS US West 1 - config.region = "us-west-1".into(); - - let integration = ObjectStore::new_amazon_s3( - Some(config.access_key_id), - Some(config.secret_access_key), - config.region, - &config.bucket, - config.endpoint, - config.token, - NonZeroUsize::new(16).unwrap(), - ) - .expect("Valid S3 config"); - - let mut location = integration.new_path(); - location.set_file_name(NON_EXISTENT_NAME); - let data = Bytes::from("arbitrary data"); - - let err = integration.put(&location, data).await.unwrap_err(); - - if let ObjectStoreError::AwsObjectStoreError { - source: - Error::UnableToPutData { - source, - bucket, - location, - }, - } = err - { - assert!(matches!(source, rusoto_core::RusotoError::Unknown(_))); - assert_eq!(bucket, config.bucket); - assert_eq!(location, NON_EXISTENT_NAME); - } else { - panic!("unexpected error type: {:?}", err); - } - } - #[tokio::test] async fn s3_test_put_nonexistent_bucket() { let mut config = maybe_skip_integration!(); @@ -954,44 +880,6 @@ mod tests { assert!(result.is_ok()); } - #[tokio::test] - async fn s3_test_delete_nonexistent_region() { - let mut config = maybe_skip_integration!(); - // Assumes environment variables do not provide credentials to AWS US West 1 - config.region = "us-west-1".into(); - - let integration = ObjectStore::new_amazon_s3( - Some(config.access_key_id), - Some(config.secret_access_key), - config.region, - &config.bucket, - config.endpoint, - config.token, - NonZeroUsize::new(16).unwrap(), - ) - .expect("Valid S3 config"); - - let mut location = integration.new_path(); - location.set_file_name(NON_EXISTENT_NAME); - - let err = integration.delete(&location).await.unwrap_err(); - if let ObjectStoreError::AwsObjectStoreError { - source: - Error::UnableToDeleteData { - source, - bucket, - location, - }, - } = err - { - assert!(matches!(source, rusoto_core::RusotoError::Unknown(_))); - assert_eq!(bucket, config.bucket); - assert_eq!(location, NON_EXISTENT_NAME); - } else { - panic!("unexpected error type: {:?}", err); - } - } - #[tokio::test] async fn s3_test_delete_nonexistent_bucket() { let mut config = maybe_skip_integration!(); From 5d58b06e64824c8e4a36ab26eaafa575abb9a7c7 Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Thu, 16 Dec 2021 11:17:36 +0100 Subject: [PATCH 2/3] test: fix some environment variables influencing our tests --- influxdb_iox/src/structopt_blocks/object_store.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/influxdb_iox/src/structopt_blocks/object_store.rs b/influxdb_iox/src/structopt_blocks/object_store.rs index e6666c5f80..f5ebe8ba5b 100644 --- a/influxdb_iox/src/structopt_blocks/object_store.rs +++ b/influxdb_iox/src/structopt_blocks/object_store.rs @@ -422,9 +422,12 @@ mod tests { #[test] fn s3_config_missing_params() { - let config = + let mut config = ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "s3"]).unwrap(); + // clean out eventual leaks via env variables + config.bucket = None; + let err = ObjectStore::try_from(&config).unwrap_err().to_string(); assert_eq!( @@ -458,9 +461,12 @@ mod tests { #[test] fn google_config_missing_params() { - let config = + let mut config = ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "google"]).unwrap(); + // clean out eventual leaks via env variables + config.bucket = None; + let err = ObjectStore::try_from(&config).unwrap_err().to_string(); assert_eq!( @@ -497,9 +503,12 @@ mod tests { #[test] fn azure_config_missing_params() { - let config = + let mut config = ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "azure"]).unwrap(); + // clean out eventual leaks via env variables + config.bucket = None; + let err = ObjectStore::try_from(&config).unwrap_err().to_string(); assert_eq!( From 36ba011653b934aa6c76470714a67e5f2cd27aee Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Thu, 16 Dec 2021 12:31:55 +0100 Subject: [PATCH 3/3] docs: explain CI docker images --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 123d360d15..8f135a533e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -168,6 +168,7 @@ jobs: } test: + # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker) docker: - image: quay.io/influxdb/rust:ci - image: localstack/localstack