Merge pull request #3388 from influxdata/crepererum/object_store_aws_test
test: `object_store` aws test using localstackpull/24376/head
kodiakhq[bot]
GitHub
commit
edcc2b5b99
|
|
@ -168,8 +168,10 @@ jobs:
|
|||
}
|
||||
|
||||
test:
|
||||
# setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
|
||||
docker:
|
||||
- image: quay.io/influxdb/rust:ci
|
||||
- image: localstack/localstack
|
||||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||||
environment:
|
||||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||||
|
|
@ -182,18 +184,31 @@ jobs:
|
|||
TEST_INTEGRATION: 1
|
||||
INFLUXDB_IOX_INTEGRATION_LOCAL: 1
|
||||
KAFKA_CONNECT: "localhost:9092"
|
||||
AWS_DEFAULT_REGION: "us-east-1"
|
||||
AWS_ACCESS_KEY_ID: test
|
||||
AWS_SECRET_ACCESS_KEY: test
|
||||
AWS_ENDPOINT: http://127.0.0.1:4566
|
||||
INFLUXDB_IOX_BUCKET: iox-test
|
||||
steps:
|
||||
- run:
|
||||
name: Run Kafka
|
||||
# Sudo needed because data directory owned by root but container runs as unprivileged user
|
||||
command: sudo /opt/redpanda/bin/redpanda --redpanda-cfg /etc/redpanda/redpanda.yaml --overprovisioned --smp 1 --memory 1G --reserve-memory 0M
|
||||
background: true
|
||||
- run:
|
||||
name: Setup localstack (AWS emulation)
|
||||
command: |
|
||||
cd /tmp
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
|
||||
unzip awscliv2.zip
|
||||
sudo ./aws/install
|
||||
aws --endpoint-url=http://localhost:4566 s3 mb s3://iox-test
|
||||
- checkout
|
||||
- rust_components
|
||||
- cache_restore
|
||||
- run:
|
||||
name: Cargo test
|
||||
command: cargo test --workspace --features=kafka
|
||||
command: cargo test --workspace --features=aws,kafka
|
||||
- cache_save
|
||||
|
||||
# end to end tests with Heappy (heap profiling enabled)
|
||||
|
|
|
|||
|
|
@ -422,9 +422,12 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn s3_config_missing_params() {
|
||||
let config =
|
||||
let mut config =
|
||||
ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "s3"]).unwrap();
|
||||
|
||||
// clean out eventual leaks via env variables
|
||||
config.bucket = None;
|
||||
|
||||
let err = ObjectStore::try_from(&config).unwrap_err().to_string();
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -458,9 +461,12 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn google_config_missing_params() {
|
||||
let config =
|
||||
let mut config =
|
||||
ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "google"]).unwrap();
|
||||
|
||||
// clean out eventual leaks via env variables
|
||||
config.bucket = None;
|
||||
|
||||
let err = ObjectStore::try_from(&config).unwrap_err().to_string();
|
||||
|
||||
assert_eq!(
|
||||
|
|
@ -497,9 +503,12 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn azure_config_missing_params() {
|
||||
let config =
|
||||
let mut config =
|
||||
ObjectStoreConfig::from_iter_safe(&["server", "--object-store", "azure"]).unwrap();
|
||||
|
||||
// clean out eventual leaks via env variables
|
||||
config.bucket = None;
|
||||
|
||||
let err = ObjectStore::try_from(&config).unwrap_err().to_string();
|
||||
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -743,40 +743,6 @@ mod tests {
|
|||
check_credentials(list_with_delimiter(&integration).await).unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_get_nonexistent_region() {
|
||||
let mut config = maybe_skip_integration!();
|
||||
// Assumes environment variables do not provide credentials to AWS US West 1
|
||||
config.region = "us-west-1".into();
|
||||
|
||||
let integration = ObjectStore::new_amazon_s3(
|
||||
Some(config.access_key_id),
|
||||
Some(config.secret_access_key),
|
||||
config.region,
|
||||
&config.bucket,
|
||||
config.endpoint,
|
||||
config.token,
|
||||
NonZeroUsize::new(16).unwrap(),
|
||||
)
|
||||
.expect("Valid S3 config");
|
||||
|
||||
let mut location = integration.new_path();
|
||||
location.set_file_name(NON_EXISTENT_NAME);
|
||||
|
||||
let err = get_nonexistent_object(&integration, Some(location))
|
||||
.await
|
||||
.unwrap_err();
|
||||
if let Some(ObjectStoreError::AwsObjectStoreError {
|
||||
source: Error::UnableToListData { source, bucket },
|
||||
}) = err.downcast_ref::<ObjectStoreError>()
|
||||
{
|
||||
assert!(matches!(source, rusoto_core::RusotoError::Unknown(_)));
|
||||
assert_eq!(bucket, &config.bucket);
|
||||
} else {
|
||||
panic!("unexpected error type: {:?}", err);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_get_nonexistent_location() {
|
||||
let config = maybe_skip_integration!();
|
||||
|
|
@ -853,46 +819,6 @@ mod tests {
|
|||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_put_nonexistent_region() {
|
||||
let mut config = maybe_skip_integration!();
|
||||
// Assumes environment variables do not provide credentials to AWS US West 1
|
||||
config.region = "us-west-1".into();
|
||||
|
||||
let integration = ObjectStore::new_amazon_s3(
|
||||
Some(config.access_key_id),
|
||||
Some(config.secret_access_key),
|
||||
config.region,
|
||||
&config.bucket,
|
||||
config.endpoint,
|
||||
config.token,
|
||||
NonZeroUsize::new(16).unwrap(),
|
||||
)
|
||||
.expect("Valid S3 config");
|
||||
|
||||
let mut location = integration.new_path();
|
||||
location.set_file_name(NON_EXISTENT_NAME);
|
||||
let data = Bytes::from("arbitrary data");
|
||||
|
||||
let err = integration.put(&location, data).await.unwrap_err();
|
||||
|
||||
if let ObjectStoreError::AwsObjectStoreError {
|
||||
source:
|
||||
Error::UnableToPutData {
|
||||
source,
|
||||
bucket,
|
||||
location,
|
||||
},
|
||||
} = err
|
||||
{
|
||||
assert!(matches!(source, rusoto_core::RusotoError::Unknown(_)));
|
||||
assert_eq!(bucket, config.bucket);
|
||||
assert_eq!(location, NON_EXISTENT_NAME);
|
||||
} else {
|
||||
panic!("unexpected error type: {:?}", err);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_put_nonexistent_bucket() {
|
||||
let mut config = maybe_skip_integration!();
|
||||
|
|
@ -954,44 +880,6 @@ mod tests {
|
|||
assert!(result.is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_delete_nonexistent_region() {
|
||||
let mut config = maybe_skip_integration!();
|
||||
// Assumes environment variables do not provide credentials to AWS US West 1
|
||||
config.region = "us-west-1".into();
|
||||
|
||||
let integration = ObjectStore::new_amazon_s3(
|
||||
Some(config.access_key_id),
|
||||
Some(config.secret_access_key),
|
||||
config.region,
|
||||
&config.bucket,
|
||||
config.endpoint,
|
||||
config.token,
|
||||
NonZeroUsize::new(16).unwrap(),
|
||||
)
|
||||
.expect("Valid S3 config");
|
||||
|
||||
let mut location = integration.new_path();
|
||||
location.set_file_name(NON_EXISTENT_NAME);
|
||||
|
||||
let err = integration.delete(&location).await.unwrap_err();
|
||||
if let ObjectStoreError::AwsObjectStoreError {
|
||||
source:
|
||||
Error::UnableToDeleteData {
|
||||
source,
|
||||
bucket,
|
||||
location,
|
||||
},
|
||||
} = err
|
||||
{
|
||||
assert!(matches!(source, rusoto_core::RusotoError::Unknown(_)));
|
||||
assert_eq!(bucket, config.bucket);
|
||||
assert_eq!(location, NON_EXISTENT_NAME);
|
||||
} else {
|
||||
panic!("unexpected error type: {:?}", err);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s3_test_delete_nonexistent_bucket() {
|
||||
let mut config = maybe_skip_integration!();
|
||||
|
|
|
|||
Loading…
Reference in New Issue