From 9f2694bf1b8e4b2af4e91325eb4adc78a0ff3ab3 Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Thu, 16 Dec 2021 13:36:02 +0100 Subject: [PATCH] test: `objest_store` azure support via Azurite --- .circleci/config.yml | 10 +++- .../src/structopt_blocks/object_store.rs | 2 +- object_store/Cargo.toml | 1 + object_store/src/azure.rs | 46 ++++++++++++++----- object_store/src/dummy.rs | 1 + object_store/src/lib.rs | 3 +- 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ad3d8addf1..692af23e20 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -172,6 +172,7 @@ jobs: docker: - image: quay.io/influxdb/rust:ci - image: localstack/localstack + - image: mcr.microsoft.com/azure-storage/azurite - image: vectorized/redpanda:v21.9.2 command: redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M resource_class: xlarge # use of a smaller executor tends crashes on link @@ -190,6 +191,7 @@ jobs: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test AWS_ENDPOINT: http://127.0.0.1:4566 + AZURE_USE_EMULATOR: "1" INFLUXDB_IOX_BUCKET: iox-test steps: - run: @@ -200,12 +202,18 @@ jobs: unzip awscliv2.zip sudo ./aws/install aws --endpoint-url=http://localhost:4566 s3 mb s3://iox-test + - run: + name: Setup Azurite (Azure emulation) + # the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings + command: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az storage container create -n iox-test --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;' - checkout - rust_components - cache_restore - run: name: Cargo test - command: cargo test --workspace --features=aws,kafka + command: cargo test --workspace --features=aws,azure,azure_test,kafka - cache_save # end to end tests with Heappy (heap profiling enabled) diff --git a/influxdb_iox/src/structopt_blocks/object_store.rs b/influxdb_iox/src/structopt_blocks/object_store.rs index f5ebe8ba5b..168fed88ae 100644 --- a/influxdb_iox/src/structopt_blocks/object_store.rs +++ b/influxdb_iox/src/structopt_blocks/object_store.rs @@ -301,7 +301,7 @@ impl TryFrom<&ObjectStoreConfig> for ObjectStore { config.azure_storage_access_key.as_ref(), ) { (Some(bucket), Some(storage_account), Some(access_key)) => { - Self::new_microsoft_azure(storage_account, access_key, bucket) + Self::new_microsoft_azure(storage_account, access_key, bucket, false) .context(InvalidAzureConfig) } (bucket, storage_account, access_key) => { diff --git a/object_store/Cargo.toml b/object_store/Cargo.toml index 0bc1eb1a59..2f0bf278fa 100644 --- a/object_store/Cargo.toml +++ b/object_store/Cargo.toml @@ -39,6 +39,7 @@ workspace-hack = { path = "../workspace-hack"} [features] azure = ["azure_core", "azure_storage", "indexmap", "reqwest"] +azure_test = ["azure", "azure_core/azurite_workaround", "azure_storage/azurite_workaround"] gcp = ["cloud-storage"] aws = ["rusoto_core", "rusoto_credential", "rusoto_s3", "hyper", "hyper-tls"] diff --git a/object_store/src/azure.rs b/object_store/src/azure.rs index 5130320c2c..91cabe8a18 100644 --- a/object_store/src/azure.rs +++ b/object_store/src/azure.rs @@ -48,6 +48,12 @@ pub enum Error { List { source: Box, }, + + #[cfg(not(feature = "azure_test"))] + #[snafu(display( + "Azurite (azure emulator) support not compiled in, please add `azure_test` feature" + ))] + NoEmulatorFeature, } /// Configuration for connecting to [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/). @@ -227,6 +233,16 @@ impl ObjectStoreApi for MicrosoftAzure { } } +#[cfg(feature = "azure_test")] +fn check_if_emulator_works() -> Result<()> { + Ok(()) +} + +#[cfg(not(feature = "azure_test"))] +fn check_if_emulator_works() -> Result<()> { + Err(Error::NoEmulatorFeature) +} + /// Configure a connection to container with given name on Microsoft Azure /// Blob store. /// @@ -236,13 +252,18 @@ pub fn new_azure( account: impl Into, access_key: impl Into, container_name: impl Into, + use_emulator: bool, ) -> Result { let account = account.into(); let access_key = access_key.into(); let http_client: Arc = Arc::new(reqwest::Client::new()); - let storage_account_client = - StorageAccountClient::new_access_key(Arc::clone(&http_client), &account, &access_key); + let storage_account_client = if use_emulator { + check_if_emulator_works()?; + StorageAccountClient::new_emulator_default() + } else { + StorageAccountClient::new_access_key(Arc::clone(&http_client), &account, &access_key) + }; let storage_client = storage_account_client.as_storage_client(); @@ -267,6 +288,7 @@ mod tests { storage_account: String, access_key: String, bucket: String, + use_emulator: bool, } // Helper macro to skip tests if TEST_INTEGRATION and the Azure environment @@ -275,11 +297,13 @@ mod tests { () => {{ dotenv::dotenv().ok(); - let required_vars = [ - "AZURE_STORAGE_ACCOUNT", - "INFLUXDB_IOX_BUCKET", - "AZURE_STORAGE_ACCESS_KEY", - ]; + let use_emulator = std::env::var("AZURE_USE_EMULATOR").is_ok(); + + let mut required_vars = vec!["INFLUXDB_IOX_BUCKET"]; + if !use_emulator { + required_vars.push("AZURE_STORAGE_ACCOUNT"); + required_vars.push("AZURE_STORAGE_ACCESS_KEY"); + } let unset_vars: Vec<_> = required_vars .iter() .filter_map(|&name| match env::var(name) { @@ -309,12 +333,11 @@ mod tests { return; } else { AzureConfig { - storage_account: env::var("AZURE_STORAGE_ACCOUNT") - .expect("already checked AZURE_STORAGE_ACCOUNT"), - access_key: env::var("AZURE_STORAGE_ACCESS_KEY") - .expect("already checked AZURE_STORAGE_ACCESS_KEY"), + storage_account: env::var("AZURE_STORAGE_ACCOUNT").unwrap_or_default(), + access_key: env::var("AZURE_STORAGE_ACCESS_KEY").unwrap_or_default(), bucket: env::var("INFLUXDB_IOX_BUCKET") .expect("already checked INFLUXDB_IOX_BUCKET"), + use_emulator, } } }}; @@ -327,6 +350,7 @@ mod tests { config.storage_account, config.access_key, config.bucket, + config.use_emulator, ) .unwrap(); diff --git a/object_store/src/dummy.rs b/object_store/src/dummy.rs index 4742e84714..e3287a33b2 100644 --- a/object_store/src/dummy.rs +++ b/object_store/src/dummy.rs @@ -116,6 +116,7 @@ pub(crate) fn new_azure( _account: impl Into, _access_key: impl Into, _container_name: impl Into, + _use_emulator: bool, ) -> Result { NotSupported { name: "azure" }.fail() } diff --git a/object_store/src/lib.rs b/object_store/src/lib.rs index ab407b6150..d804a0c5c7 100644 --- a/object_store/src/lib.rs +++ b/object_store/src/lib.rs @@ -190,8 +190,9 @@ impl ObjectStore { account: impl Into, access_key: impl Into, container_name: impl Into, + use_emulator: bool, ) -> Result { - let azure = azure::new_azure(account, access_key, container_name)?; + let azure = azure::new_azure(account, access_key, container_name, use_emulator)?; Ok(Self { integration: ObjectStoreIntegration::MicrosoftAzure(Box::new(azure)), cache: None,