test: `objest_store` azure support via Azurite

.circleci/config.yml

@@ -172,6 +172,7 @@ jobs:
     docker:
       - image: quay.io/influxdb/rust:ci
       - image: localstack/localstack
+      - image: mcr.microsoft.com/azure-storage/azurite
       - image: vectorized/redpanda:v21.9.2
         command: redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M
     resource_class: xlarge  # use of a smaller executor tends crashes on link
@@ -190,6 +191,7 @@ jobs:
       AWS_ACCESS_KEY_ID: test
       AWS_SECRET_ACCESS_KEY: test
       AWS_ENDPOINT: http://127.0.0.1:4566
+      AZURE_USE_EMULATOR: "1"
       INFLUXDB_IOX_BUCKET: iox-test
     steps:
       - run:
@@ -200,12 +202,18 @@ jobs:
             unzip awscliv2.zip
             sudo ./aws/install
             aws --endpoint-url=http://localhost:4566 s3 mb s3://iox-test
+      - run:
+          name: Setup Azurite (Azure emulation)
+          # the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
+          command: |
+            curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+            az storage container create -n iox-test --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;'
       - checkout
       - rust_components
       - cache_restore
       - run:
           name: Cargo test
-          command: cargo test --workspace --features=aws,kafka
+          command: cargo test --workspace --features=aws,azure,azure_test,kafka
       - cache_save
 
   # end to end tests with Heappy (heap profiling enabled)

influxdb_iox/src/structopt_blocks/object_store.rs

@@ -301,7 +301,7 @@ impl TryFrom<&ObjectStoreConfig> for ObjectStore {
                     config.azure_storage_access_key.as_ref(),
                 ) {
                     (Some(bucket), Some(storage_account), Some(access_key)) => {
-                        Self::new_microsoft_azure(storage_account, access_key, bucket)
+                        Self::new_microsoft_azure(storage_account, access_key, bucket, false)
                             .context(InvalidAzureConfig)
                     }
                     (bucket, storage_account, access_key) => {

object_store/Cargo.toml

@@ -39,6 +39,7 @@ workspace-hack = { path = "../workspace-hack"}
 
 [features]
 azure = ["azure_core", "azure_storage", "indexmap", "reqwest"]
+azure_test = ["azure", "azure_core/azurite_workaround", "azure_storage/azurite_workaround"]
 gcp = ["cloud-storage"]
 aws = ["rusoto_core", "rusoto_credential", "rusoto_s3", "hyper", "hyper-tls"]
 

object_store/src/azure.rs

@@ -48,6 +48,12 @@ pub enum Error {
     List {
         source: Box<dyn std::error::Error + Send + Sync>,
     },
+
+    #[cfg(not(feature = "azure_test"))]
+    #[snafu(display(
+        "Azurite (azure emulator) support not compiled in, please add `azure_test` feature"
+    ))]
+    NoEmulatorFeature,
 }
 
 /// Configuration for connecting to [Microsoft Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/).
@@ -227,6 +233,16 @@ impl ObjectStoreApi for MicrosoftAzure {
     }
 }
 
+#[cfg(feature = "azure_test")]
+fn check_if_emulator_works() -> Result<()> {
+    Ok(())
+}
+
+#[cfg(not(feature = "azure_test"))]
+fn check_if_emulator_works() -> Result<()> {
+    Err(Error::NoEmulatorFeature)
+}
+
 /// Configure a connection to container with given name on Microsoft Azure
 /// Blob store.
 ///
@@ -236,13 +252,18 @@ pub fn new_azure(
     account: impl Into<String>,
     access_key: impl Into<String>,
     container_name: impl Into<String>,
+    use_emulator: bool,
 ) -> Result<MicrosoftAzure> {
     let account = account.into();
     let access_key = access_key.into();
     let http_client: Arc<dyn HttpClient> = Arc::new(reqwest::Client::new());
 
-    let storage_account_client =
-        StorageAccountClient::new_access_key(Arc::clone(&http_client), &account, &access_key);
+    let storage_account_client = if use_emulator {
+        check_if_emulator_works()?;
+        StorageAccountClient::new_emulator_default()
+    } else {
+        StorageAccountClient::new_access_key(Arc::clone(&http_client), &account, &access_key)
+    };
 
     let storage_client = storage_account_client.as_storage_client();
 
@@ -267,6 +288,7 @@ mod tests {
         storage_account: String,
         access_key: String,
         bucket: String,
+        use_emulator: bool,
     }
 
     // Helper macro to skip tests if TEST_INTEGRATION and the Azure environment
@@ -275,11 +297,13 @@ mod tests {
         () => {{
             dotenv::dotenv().ok();
 
-            let required_vars = [
-                "AZURE_STORAGE_ACCOUNT",
-                "INFLUXDB_IOX_BUCKET",
-                "AZURE_STORAGE_ACCESS_KEY",
-            ];
+            let use_emulator = std::env::var("AZURE_USE_EMULATOR").is_ok();
+
+            let mut required_vars = vec!["INFLUXDB_IOX_BUCKET"];
+            if !use_emulator {
+                required_vars.push("AZURE_STORAGE_ACCOUNT");
+                required_vars.push("AZURE_STORAGE_ACCESS_KEY");
+            }
             let unset_vars: Vec<_> = required_vars
                 .iter()
                 .filter_map(|&name| match env::var(name) {
@@ -309,12 +333,11 @@ mod tests {
                 return;
             } else {
                 AzureConfig {
-                    storage_account: env::var("AZURE_STORAGE_ACCOUNT")
-                        .expect("already checked AZURE_STORAGE_ACCOUNT"),
-                    access_key: env::var("AZURE_STORAGE_ACCESS_KEY")
-                        .expect("already checked AZURE_STORAGE_ACCESS_KEY"),
+                    storage_account: env::var("AZURE_STORAGE_ACCOUNT").unwrap_or_default(),
+                    access_key: env::var("AZURE_STORAGE_ACCESS_KEY").unwrap_or_default(),
                     bucket: env::var("INFLUXDB_IOX_BUCKET")
                         .expect("already checked INFLUXDB_IOX_BUCKET"),
+                    use_emulator,
                 }
             }
         }};
@@ -327,6 +350,7 @@ mod tests {
             config.storage_account,
             config.access_key,
             config.bucket,
+            config.use_emulator,
         )
         .unwrap();
 

object_store/src/dummy.rs

@@ -116,6 +116,7 @@ pub(crate) fn new_azure(
     _account: impl Into<String>,
     _access_key: impl Into<String>,
     _container_name: impl Into<String>,
+    _use_emulator: bool,
 ) -> Result<DummyObjectStore> {
     NotSupported { name: "azure" }.fail()
 }

object_store/src/lib.rs

@@ -190,8 +190,9 @@ impl ObjectStore {
         account: impl Into<String>,
         access_key: impl Into<String>,
         container_name: impl Into<String>,
+        use_emulator: bool,
     ) -> Result<Self> {
-        let azure = azure::new_azure(account, access_key, container_name)?;
+        let azure = azure::new_azure(account, access_key, container_name, use_emulator)?;
         Ok(Self {
             integration: ObjectStoreIntegration::MicrosoftAzure(Box::new(azure)),
             cache: None,