Merge pull request #1416 from DCjanus/main

feat: s3 compatibility storages support

docs/env.example

@@ -34,6 +34,9 @@
 # AWS_DEFAULT_REGION=us-east-2
 # INFLUXDB_IOX_BUCKET=bucket-name
 #
+# If using an s3 compatible storage
+# AWS_ENDPOINT = http://localhost:9000
+#
 # If using Google Cloud Storage as an object store:
 # INFLUXDB_IOX_BUCKET=bucket_name
 # Set one of SERVICE_ACCOUNT or GOOGLE_APPLICATION_CREDENTIALS, either to a path of a filename

object_store/src/aws.rs

@@ -295,9 +295,16 @@ impl AmazonS3 {
         secret_access_key: Option<impl Into<String>>,
         region: impl Into<String>,
         bucket_name: impl Into<String>,
+        endpoint: Option<impl Into<String>>,
     ) -> Result<Self> {
         let region = region.into();
-        let region: rusoto_core::Region = region.parse().context(InvalidRegion { region })?;
+        let region: rusoto_core::Region = match endpoint {
+            None => region.parse().context(InvalidRegion { region })?,
+            Some(endpoint) => rusoto_core::Region::Custom {
+                name: region,
+                endpoint: endpoint.into(),
+            },
+        };
 
         let http_client = rusoto_core::request::HttpClient::new()
             .expect("Current implementation of rusoto_core has no way for this to fail");
@@ -523,6 +530,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -543,6 +551,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -573,6 +582,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -614,6 +624,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -650,6 +661,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -696,6 +708,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -740,6 +753,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -764,6 +778,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );
@@ -800,6 +815,7 @@ mod tests {
                 Some(config.secret_access_key),
                 config.region,
                 &config.bucket,
+                Option::<String>::None,
             )
             .expect("Valid S3 config"),
         );

src/commands/run.rs

@@ -361,6 +361,17 @@ Possible values (case insensitive):
     )]
     pub aws_default_region: String,
 
+    /// When using Amazon s3 compatibility storage service, set this to the
+    /// endpoint.
+    ///
+    /// Must also set `--object-store=s3`, `--bucket`. Can also set `--aws-default-region`
+    /// if not using the fallback region.
+    ///
+    /// Prefer the environment variable over the command line flag in shared
+    /// environments.
+    #[structopt(long = "--aws-endpoint", env = "AWS_ENDPOINT")]
+    pub aws_endpoint: Option<String>,
+
     /// When using Google Cloud Storage as the object store, set this to the
     /// path to the JSON file that contains the Google credentials.
     ///

src/influxdb_ioxd.rs

@@ -319,12 +319,15 @@ impl TryFrom<&Config> for ObjectStore {
                     config.aws_access_key_id.as_ref(),
                     config.aws_secret_access_key.as_ref(),
                     config.aws_default_region.as_str(),
+                    config.aws_endpoint.as_ref(),
                 ) {
-                    (Some(bucket), key_id, secret_key, region) => Ok(Self::new_amazon_s3(
-                        AmazonS3::new(key_id, secret_key, region, bucket)
-                            .context(InvalidS3Config)?,
-                    )),
-                    (bucket, _, _, _) => {
+                    (Some(bucket), key_id, secret_key, region, endpoint) => {
+                        Ok(Self::new_amazon_s3(
+                            AmazonS3::new(key_id, secret_key, region, bucket, endpoint)
+                                .context(InvalidS3Config)?,
+                        ))
+                    }
+                    (bucket, _, _, _, _) => {
                         let mut missing_args = vec![];
 
                         if bucket.is_none() {