fix: Remove database delete/restore entirely
parent
7783e4a7ff
commit
f69d37e9a8
|
@ -49,15 +49,9 @@ service ManagementService {
|
||||||
// Roughly follows the <https://google.aip.dev/134> pattern, except we wrap the response
|
// Roughly follows the <https://google.aip.dev/134> pattern, except we wrap the response
|
||||||
rpc UpdateDatabase(UpdateDatabaseRequest) returns (UpdateDatabaseResponse);
|
rpc UpdateDatabase(UpdateDatabaseRequest) returns (UpdateDatabaseResponse);
|
||||||
|
|
||||||
// Delete a database.
|
|
||||||
rpc DeleteDatabase(DeleteDatabaseRequest) returns (DeleteDatabaseResponse);
|
|
||||||
|
|
||||||
// Release a database from its current server.
|
// Release a database from its current server.
|
||||||
rpc ReleaseDatabase(ReleaseDatabaseRequest) returns (ReleaseDatabaseResponse);
|
rpc ReleaseDatabase(ReleaseDatabaseRequest) returns (ReleaseDatabaseResponse);
|
||||||
|
|
||||||
// Restore a deleted database.
|
|
||||||
rpc RestoreDatabase(RestoreDatabaseRequest) returns (RestoreDatabaseResponse);
|
|
||||||
|
|
||||||
// Claim a released database.
|
// Claim a released database.
|
||||||
rpc ClaimDatabase(ClaimDatabaseRequest) returns (ClaimDatabaseResponse);
|
rpc ClaimDatabase(ClaimDatabaseRequest) returns (ClaimDatabaseResponse);
|
||||||
|
|
||||||
|
@ -209,15 +203,6 @@ message UpdateDatabaseResponse {
|
||||||
DatabaseRules rules = 1;
|
DatabaseRules rules = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
message DeleteDatabaseRequest {
|
|
||||||
// the name of the database
|
|
||||||
string db_name = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
message DeleteDatabaseResponse {
|
|
||||||
bytes uuid = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
message ReleaseDatabaseRequest {
|
message ReleaseDatabaseRequest {
|
||||||
// the name of the database
|
// the name of the database
|
||||||
string db_name = 1;
|
string db_name = 1;
|
||||||
|
@ -230,24 +215,6 @@ message ReleaseDatabaseResponse {
|
||||||
bytes uuid = 1;
|
bytes uuid = 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
message RestoreDatabaseRequest {
|
|
||||||
// Was the generation ID of the deleted database.
|
|
||||||
reserved 1;
|
|
||||||
reserved "generation_id";
|
|
||||||
|
|
||||||
// Was the name of the database
|
|
||||||
reserved 2;
|
|
||||||
reserved "db_name";
|
|
||||||
|
|
||||||
// Was the string-formatted UUID of the deleted database.
|
|
||||||
reserved 3;
|
|
||||||
|
|
||||||
// The UUID of the deleted database.
|
|
||||||
bytes uuid = 4;
|
|
||||||
}
|
|
||||||
|
|
||||||
message RestoreDatabaseResponse {}
|
|
||||||
|
|
||||||
message ClaimDatabaseRequest {
|
message ClaimDatabaseRequest {
|
||||||
bytes uuid = 1;
|
bytes uuid = 1;
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,8 +7,8 @@ use influxdb_iox_client::{
|
||||||
flight,
|
flight,
|
||||||
format::QueryOutputFormat,
|
format::QueryOutputFormat,
|
||||||
management::{
|
management::{
|
||||||
self, generated_types::*, ClaimDatabaseError, CreateDatabaseError, DeleteDatabaseError,
|
self, generated_types::*, ClaimDatabaseError, CreateDatabaseError, GetDatabaseError,
|
||||||
GetDatabaseError, ListDatabaseError, ReleaseDatabaseError, RestoreDatabaseError,
|
ListDatabaseError, ReleaseDatabaseError,
|
||||||
},
|
},
|
||||||
write::{self, WriteError},
|
write::{self, WriteError},
|
||||||
};
|
};
|
||||||
|
@ -34,15 +34,9 @@ pub enum Error {
|
||||||
#[error("Error listing databases: {0}")]
|
#[error("Error listing databases: {0}")]
|
||||||
ListDatabaseError(#[from] ListDatabaseError),
|
ListDatabaseError(#[from] ListDatabaseError),
|
||||||
|
|
||||||
#[error("Error deleting database: {0}")]
|
|
||||||
DeleteDatabaseError(#[from] DeleteDatabaseError),
|
|
||||||
|
|
||||||
#[error("Error releasing database: {0}")]
|
#[error("Error releasing database: {0}")]
|
||||||
ReleaseDatabaseError(#[from] ReleaseDatabaseError),
|
ReleaseDatabaseError(#[from] ReleaseDatabaseError),
|
||||||
|
|
||||||
#[error("Error restoring database: {0}")]
|
|
||||||
RestoreDatabaseError(#[from] RestoreDatabaseError),
|
|
||||||
|
|
||||||
#[error("Error claiming database: {0}")]
|
#[error("Error claiming database: {0}")]
|
||||||
ClaimDatabaseError(#[from] ClaimDatabaseError),
|
ClaimDatabaseError(#[from] ClaimDatabaseError),
|
||||||
|
|
||||||
|
@ -186,13 +180,6 @@ struct Query {
|
||||||
format: String,
|
format: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete a database
|
|
||||||
#[derive(Debug, StructOpt)]
|
|
||||||
struct Delete {
|
|
||||||
/// The name of the database to delete
|
|
||||||
name: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Release a database from its current server owner
|
/// Release a database from its current server owner
|
||||||
#[derive(Debug, StructOpt)]
|
#[derive(Debug, StructOpt)]
|
||||||
struct Release {
|
struct Release {
|
||||||
|
@ -205,13 +192,6 @@ struct Release {
|
||||||
uuid: Option<Uuid>,
|
uuid: Option<Uuid>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restore a deleted database
|
|
||||||
#[derive(Debug, StructOpt)]
|
|
||||||
struct Restore {
|
|
||||||
/// The UUID of the database to restore
|
|
||||||
uuid: Uuid,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Claim an unowned database
|
/// Claim an unowned database
|
||||||
#[derive(Debug, StructOpt)]
|
#[derive(Debug, StructOpt)]
|
||||||
struct Claim {
|
struct Claim {
|
||||||
|
@ -230,9 +210,7 @@ enum Command {
|
||||||
Chunk(chunk::Config),
|
Chunk(chunk::Config),
|
||||||
Partition(partition::Config),
|
Partition(partition::Config),
|
||||||
Recover(recover::Config),
|
Recover(recover::Config),
|
||||||
Delete(Delete),
|
|
||||||
Release(Release),
|
Release(Release),
|
||||||
Restore(Restore),
|
|
||||||
Claim(Claim),
|
Claim(Claim),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -369,23 +347,12 @@ pub async fn command(connection: Connection, config: Config) -> Result<()> {
|
||||||
Command::Recover(config) => {
|
Command::Recover(config) => {
|
||||||
recover::command(connection, config).await?;
|
recover::command(connection, config).await?;
|
||||||
}
|
}
|
||||||
Command::Delete(command) => {
|
|
||||||
let mut client = management::Client::new(connection);
|
|
||||||
let uuid = client.delete_database(&command.name).await?;
|
|
||||||
println!("Deleted database {}", command.name);
|
|
||||||
println!("{}", uuid);
|
|
||||||
}
|
|
||||||
Command::Release(command) => {
|
Command::Release(command) => {
|
||||||
let mut client = management::Client::new(connection);
|
let mut client = management::Client::new(connection);
|
||||||
let uuid = client.release_database(&command.name, command.uuid).await?;
|
let uuid = client.release_database(&command.name, command.uuid).await?;
|
||||||
println!("Released database {}", command.name);
|
println!("Released database {}", command.name);
|
||||||
println!("{}", uuid);
|
println!("{}", uuid);
|
||||||
}
|
}
|
||||||
Command::Restore(command) => {
|
|
||||||
let mut client = management::Client::new(connection);
|
|
||||||
client.restore_database(command.uuid).await?;
|
|
||||||
println!("Restored database {}", command.uuid);
|
|
||||||
}
|
|
||||||
Command::Claim(command) => {
|
Command::Claim(command) => {
|
||||||
let mut client = management::Client::new(connection);
|
let mut client = management::Client::new(connection);
|
||||||
let db_name = client.claim_database(command.uuid).await?;
|
let db_name = client.claim_database(command.uuid).await?;
|
||||||
|
|
|
@ -57,9 +57,7 @@ pub fn default_server_error_handler(error: server::Error) -> tonic::Status {
|
||||||
}
|
}
|
||||||
.into(),
|
.into(),
|
||||||
Error::RemoteError { source } => tonic::Status::unavailable(source.to_string()),
|
Error::RemoteError { source } => tonic::Status::unavailable(source.to_string()),
|
||||||
Error::WipePreservedCatalog { source } | Error::CannotMarkDatabaseDeleted { source } => {
|
Error::WipePreservedCatalog { source } => default_database_error_handler(source),
|
||||||
default_database_error_handler(source)
|
|
||||||
}
|
|
||||||
Error::DeleteExpression {
|
Error::DeleteExpression {
|
||||||
start_time,
|
start_time,
|
||||||
stop_time,
|
stop_time,
|
||||||
|
@ -76,12 +74,7 @@ pub fn default_server_error_handler(error: server::Error) -> tonic::Status {
|
||||||
tonic::Status::invalid_argument(format!("Cannot initialize database: {}", source))
|
tonic::Status::invalid_argument(format!("Cannot initialize database: {}", source))
|
||||||
}
|
}
|
||||||
Error::StoreWriteErrors { .. } => tonic::Status::invalid_argument(error.to_string()),
|
Error::StoreWriteErrors { .. } => tonic::Status::invalid_argument(error.to_string()),
|
||||||
Error::CannotRestoreDatabase {
|
Error::DatabaseAlreadyExists { .. } | Error::DatabaseAlreadyOwnedByThisServer { .. } => {
|
||||||
source: e @ server::database::InitError::AlreadyActive { .. },
|
|
||||||
} => tonic::Status::already_exists(e.to_string()),
|
|
||||||
Error::DatabaseAlreadyActive { .. }
|
|
||||||
| Error::DatabaseAlreadyExists { .. }
|
|
||||||
| Error::DatabaseAlreadyOwnedByThisServer { .. } => {
|
|
||||||
tonic::Status::already_exists(error.to_string())
|
tonic::Status::already_exists(error.to_string())
|
||||||
}
|
}
|
||||||
Error::UuidMismatch { .. } | Error::CannotClaimDatabase { .. } => {
|
Error::UuidMismatch { .. } | Error::CannotClaimDatabase { .. } => {
|
||||||
|
@ -146,13 +139,7 @@ pub fn default_database_error_handler(error: server::database::Error) -> tonic::
|
||||||
error!(%source, "Unexpected error skipping replay");
|
error!(%source, "Unexpected error skipping replay");
|
||||||
InternalError {}.into()
|
InternalError {}.into()
|
||||||
}
|
}
|
||||||
Error::CannotMarkDatabaseDeleted { source, .. } => {
|
Error::CannotReleaseUnowned { .. } => tonic::Status::failed_precondition(error.to_string()),
|
||||||
error!(%source, "Unexpected error deleting database");
|
|
||||||
InternalError {}.into()
|
|
||||||
}
|
|
||||||
Error::CannotDeleteInactiveDatabase { .. } | Error::CannotReleaseUnowned { .. } => {
|
|
||||||
tonic::Status::failed_precondition(error.to_string())
|
|
||||||
}
|
|
||||||
Error::CannotRelease { source, .. } => {
|
Error::CannotRelease { source, .. } => {
|
||||||
error!(%source, "Unexpected error releasing database");
|
error!(%source, "Unexpected error releasing database");
|
||||||
InternalError {}.into()
|
InternalError {}.into()
|
||||||
|
|
|
@ -169,23 +169,6 @@ where
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn delete_database(
|
|
||||||
&self,
|
|
||||||
request: Request<DeleteDatabaseRequest>,
|
|
||||||
) -> Result<Response<DeleteDatabaseResponse>, Status> {
|
|
||||||
let db_name = DatabaseName::new(request.into_inner().db_name).scope("db_name")?;
|
|
||||||
|
|
||||||
let uuid = self
|
|
||||||
.server
|
|
||||||
.release_database(&db_name, None)
|
|
||||||
.await
|
|
||||||
.map_err(default_server_error_handler)?;
|
|
||||||
|
|
||||||
Ok(Response::new(DeleteDatabaseResponse {
|
|
||||||
uuid: uuid.as_bytes().to_vec(),
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn release_database(
|
async fn release_database(
|
||||||
&self,
|
&self,
|
||||||
request: Request<ReleaseDatabaseRequest>,
|
request: Request<ReleaseDatabaseRequest>,
|
||||||
|
@ -210,21 +193,6 @@ where
|
||||||
}))
|
}))
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn restore_database(
|
|
||||||
&self,
|
|
||||||
request: Request<RestoreDatabaseRequest>,
|
|
||||||
) -> Result<Response<RestoreDatabaseResponse>, Status> {
|
|
||||||
let request = request.into_inner();
|
|
||||||
let uuid = Uuid::from_slice(&request.uuid).scope("uuid")?;
|
|
||||||
|
|
||||||
self.server
|
|
||||||
.claim_database(uuid)
|
|
||||||
.await
|
|
||||||
.map_err(default_server_error_handler)?;
|
|
||||||
|
|
||||||
Ok(Response::new(RestoreDatabaseResponse {}))
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn claim_database(
|
async fn claim_database(
|
||||||
&self,
|
&self,
|
||||||
request: Request<ClaimDatabaseRequest>,
|
request: Request<ClaimDatabaseRequest>,
|
||||||
|
|
|
@ -44,9 +44,12 @@ async fn test_querying_deleted_database() {
|
||||||
];
|
];
|
||||||
assert_batches_sorted_eq!(&expected, &batches);
|
assert_batches_sorted_eq!(&expected, &batches);
|
||||||
|
|
||||||
// Ensure we get an error after deleting the database
|
// Ensure we get an error after releasing the database
|
||||||
|
|
||||||
management_client.delete_database(&db_name).await.unwrap();
|
management_client
|
||||||
|
.release_database(&db_name, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
assert!(query_cpu_to_batches(&mut flight_client, &db_name)
|
assert!(query_cpu_to_batches(&mut flight_client, &db_name)
|
||||||
.await
|
.await
|
||||||
|
|
|
@ -288,8 +288,8 @@ async fn test_list_databases() {
|
||||||
assert!(rules.lifecycle_rules.is_none());
|
assert!(rules.lifecycle_rules.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
// now delete one of the databases; it should not appear whether we're omitting defaults or not
|
// now release one of the databases; it should not appear whether we're omitting defaults or not
|
||||||
client.delete_database(&name1).await.unwrap();
|
client.release_database(&name1, None).await.unwrap();
|
||||||
|
|
||||||
let omit_defaults = false;
|
let omit_defaults = false;
|
||||||
let databases: Vec<_> = client
|
let databases: Vec<_> = client
|
||||||
|
@ -324,7 +324,7 @@ async fn test_list_databases() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_create_get_update_delete_restore_database() {
|
async fn test_create_get_update_release_claim_database() {
|
||||||
test_helpers::maybe_start_logging();
|
test_helpers::maybe_start_logging();
|
||||||
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
||||||
let mut client = server_fixture.management_client();
|
let mut client = server_fixture.management_client();
|
||||||
|
@ -410,66 +410,42 @@ async fn test_create_get_update_delete_restore_database() {
|
||||||
assert_eq!(databases.len(), 1);
|
assert_eq!(databases.len(), 1);
|
||||||
assert_eq!(Uuid::from_slice(&databases[0].uuid).unwrap(), created_uuid);
|
assert_eq!(Uuid::from_slice(&databases[0].uuid).unwrap(), created_uuid);
|
||||||
|
|
||||||
let deleted_uuid = client
|
let released_uuid = client.release_database(&db_name, None).await.unwrap();
|
||||||
.delete_database(&db_name)
|
assert_eq!(created_uuid, released_uuid);
|
||||||
.await
|
|
||||||
.expect("delete database failed");
|
|
||||||
assert_eq!(created_uuid, deleted_uuid);
|
|
||||||
|
|
||||||
let err = client
|
let err = client.get_database(&db_name, false).await.unwrap_err();
|
||||||
.get_database(&db_name, false)
|
|
||||||
.await
|
|
||||||
.expect_err("get database should have failed but didn't");
|
|
||||||
assert_contains!(err.to_string(), "Database not found");
|
assert_contains!(err.to_string(), "Database not found");
|
||||||
|
|
||||||
client
|
client.claim_database(released_uuid).await.unwrap();
|
||||||
.restore_database(deleted_uuid)
|
|
||||||
.await
|
|
||||||
.expect("restore database failed");
|
|
||||||
|
|
||||||
client
|
client.get_database(&db_name, false).await.unwrap();
|
||||||
.get_database(&db_name, false)
|
|
||||||
.await
|
|
||||||
.expect("get database failed");
|
|
||||||
|
|
||||||
let err = client
|
let err = client.claim_database(released_uuid).await.unwrap_err();
|
||||||
.restore_database(deleted_uuid)
|
|
||||||
.await
|
|
||||||
.expect_err("restore database should have failed but didn't");
|
|
||||||
assert_contains!(
|
assert_contains!(
|
||||||
err.to_string(),
|
err.to_string(),
|
||||||
format!(
|
format!(
|
||||||
"The database with UUID `{}` is already owned by this server",
|
"The database with UUID `{}` is already owned by this server",
|
||||||
deleted_uuid
|
released_uuid
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
|
|
||||||
let unknown_uuid = Uuid::new_v4();
|
let unknown_uuid = Uuid::new_v4();
|
||||||
let err = client
|
let err = client.claim_database(unknown_uuid).await.unwrap_err();
|
||||||
.restore_database(unknown_uuid)
|
|
||||||
.await
|
|
||||||
.expect_err("restore database should have failed but didn't");
|
|
||||||
assert_contains!(
|
assert_contains!(
|
||||||
err.to_string(),
|
err.to_string(),
|
||||||
format!("Could not find a database with UUID `{}`", unknown_uuid)
|
format!("Could not find a database with UUID `{}`", unknown_uuid)
|
||||||
);
|
);
|
||||||
|
|
||||||
client
|
client.release_database(&db_name, None).await.unwrap();
|
||||||
.delete_database(&db_name)
|
|
||||||
.await
|
|
||||||
.expect("delete database failed");
|
|
||||||
|
|
||||||
let newly_created_uuid = client
|
let newly_created_uuid = client
|
||||||
.create_database(rules.clone())
|
.create_database(rules.clone())
|
||||||
.await
|
.await
|
||||||
.expect("create database failed");
|
.expect("create database failed");
|
||||||
|
|
||||||
assert_ne!(deleted_uuid, newly_created_uuid);
|
assert_ne!(released_uuid, newly_created_uuid);
|
||||||
|
|
||||||
let err = client
|
let err = client.claim_database(released_uuid).await.unwrap_err();
|
||||||
.restore_database(deleted_uuid)
|
|
||||||
.await
|
|
||||||
.expect_err("restore database should have failed but didn't");
|
|
||||||
assert_contains!(
|
assert_contains!(
|
||||||
err.to_string(),
|
err.to_string(),
|
||||||
format!("A database with the name `{}` already exists", db_name)
|
format!("A database with the name `{}` already exists", db_name)
|
||||||
|
|
|
@ -183,7 +183,7 @@ async fn test_create_database_immutable() {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn delete_restore_database() {
|
async fn release_claim_database() {
|
||||||
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
||||||
let addr = server_fixture.grpc_base();
|
let addr = server_fixture.grpc_base();
|
||||||
let db_name = rand_name();
|
let db_name = rand_name();
|
||||||
|
@ -223,128 +223,29 @@ async fn delete_restore_database() {
|
||||||
.success()
|
.success()
|
||||||
.stdout(predicate::str::contains(db));
|
.stdout(predicate::str::contains(db));
|
||||||
|
|
||||||
// Delete the database, returns the UUID
|
// Release the database, returns the UUID
|
||||||
let stdout = String::from_utf8(
|
let stdout = String::from_utf8(
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.arg("database")
|
.arg("database")
|
||||||
.arg("delete")
|
.arg("release")
|
||||||
.arg(db)
|
.arg(db)
|
||||||
.arg("--host")
|
.arg("--host")
|
||||||
.arg(addr)
|
.arg(addr)
|
||||||
.assert()
|
.assert()
|
||||||
.success()
|
.success()
|
||||||
.stdout(predicate::str::contains(format!("Deleted database {}", db)))
|
|
||||||
.get_output()
|
|
||||||
.stdout
|
|
||||||
.clone(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let db_uuid = stdout.lines().last().unwrap().trim();
|
|
||||||
|
|
||||||
// Listing the databases does not include the deleted database
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("list")
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(db).not());
|
|
||||||
|
|
||||||
// Deleting the database again is an error
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("delete")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.failure()
|
|
||||||
.stderr(predicate::str::contains(
|
|
||||||
"Error deleting database: Database not found",
|
|
||||||
));
|
|
||||||
|
|
||||||
// Creating a new database with the same name works
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("create")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains("Created"));
|
|
||||||
|
|
||||||
// The newly-created database will be in the active list
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("list")
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(db));
|
|
||||||
|
|
||||||
// Restoring the 1st database is an error because the new, currently active database has the
|
|
||||||
// same name
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("restore")
|
|
||||||
.arg(db_uuid)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.failure()
|
|
||||||
.stderr(predicate::str::contains(format!(
|
|
||||||
"A database with the name `{}` already exists",
|
|
||||||
db
|
|
||||||
)));
|
|
||||||
|
|
||||||
// Delete the 2nd database
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("delete")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!("Deleted database {}", db)));
|
|
||||||
|
|
||||||
// The 2nd database should no longer be in the active list
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("list")
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(db).not());
|
|
||||||
|
|
||||||
// Restore the 1st database
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("restore")
|
|
||||||
.arg(db_uuid)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!(
|
.stdout(predicate::str::contains(format!(
|
||||||
"Restored database {}",
|
"Released database {}",
|
||||||
db_uuid
|
db
|
||||||
)));
|
)))
|
||||||
|
.get_output()
|
||||||
|
.stdout
|
||||||
|
.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let db_uuid = stdout.lines().last().unwrap().trim();
|
||||||
|
|
||||||
// The 1st database is back in the active list
|
// Listing the databases does not include the released database
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.arg("database")
|
.arg("database")
|
||||||
|
@ -353,79 +254,23 @@ async fn delete_restore_database() {
|
||||||
.arg(addr)
|
.arg(addr)
|
||||||
.assert()
|
.assert()
|
||||||
.success()
|
.success()
|
||||||
.stdout(predicate::str::contains(db));
|
.stdout(predicate::str::contains(db).not());
|
||||||
|
|
||||||
// Restoring again is an error
|
// Releasing the database again is an error
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.arg("database")
|
.arg("database")
|
||||||
.arg("restore")
|
.arg("release")
|
||||||
.arg(db_uuid)
|
.arg(db)
|
||||||
.arg("--host")
|
.arg("--host")
|
||||||
.arg(addr)
|
.arg(addr)
|
||||||
.assert()
|
.assert()
|
||||||
.failure()
|
.failure()
|
||||||
.stderr(predicate::str::contains(format!(
|
.stderr(predicate::str::contains(format!(
|
||||||
"The database with UUID `{}` is already owned by this server",
|
"Error releasing database: Could not find database {}",
|
||||||
db_uuid
|
db
|
||||||
)));
|
)));
|
||||||
|
|
||||||
// Restoring a database with a valid but unknown UUID is an error
|
|
||||||
let unknown_uuid = Uuid::new_v4();
|
|
||||||
dbg!(unknown_uuid);
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("restore")
|
|
||||||
.arg(unknown_uuid.to_string())
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.failure()
|
|
||||||
.stderr(predicate::str::contains(format!(
|
|
||||||
"Could not find a database with UUID `{}`",
|
|
||||||
unknown_uuid
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure that "database delete" works with "database claim"
|
|
||||||
#[tokio::test]
|
|
||||||
async fn delete_claim_database() {
|
|
||||||
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
|
||||||
let addr = server_fixture.grpc_base();
|
|
||||||
let db_name = rand_name();
|
|
||||||
let db = &db_name;
|
|
||||||
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("create")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains("Created"));
|
|
||||||
|
|
||||||
// Delete the database, returns the UUID
|
|
||||||
let stdout = String::from_utf8(
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("delete")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!("Deleted database {}", db)))
|
|
||||||
.get_output()
|
|
||||||
.stdout
|
|
||||||
.clone(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let db_uuid = stdout.lines().last().unwrap().trim();
|
|
||||||
|
|
||||||
// Creating a new database with the same name works
|
// Creating a new database with the same name works
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
|
@ -465,17 +310,31 @@ async fn delete_claim_database() {
|
||||||
db
|
db
|
||||||
)));
|
)));
|
||||||
|
|
||||||
// Delete the 2nd database
|
// Release the 2nd database
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.arg("database")
|
.arg("database")
|
||||||
.arg("delete")
|
.arg("release")
|
||||||
.arg(db)
|
.arg(db)
|
||||||
.arg("--host")
|
.arg("--host")
|
||||||
.arg(addr)
|
.arg(addr)
|
||||||
.assert()
|
.assert()
|
||||||
.success()
|
.success()
|
||||||
.stdout(predicate::str::contains(format!("Deleted database {}", db)));
|
.stdout(predicate::str::contains(format!(
|
||||||
|
"Released database {}",
|
||||||
|
db
|
||||||
|
)));
|
||||||
|
|
||||||
|
// The 2nd database should no longer be in the active list
|
||||||
|
Command::cargo_bin("influxdb_iox")
|
||||||
|
.unwrap()
|
||||||
|
.arg("database")
|
||||||
|
.arg("list")
|
||||||
|
.arg("--host")
|
||||||
|
.arg(addr)
|
||||||
|
.assert()
|
||||||
|
.success()
|
||||||
|
.stdout(predicate::str::contains(db).not());
|
||||||
|
|
||||||
// Claim the 1st database
|
// Claim the 1st database
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
|
@ -514,117 +373,23 @@ async fn delete_claim_database() {
|
||||||
"The database with UUID `{}` is already owned by this server",
|
"The database with UUID `{}` is already owned by this server",
|
||||||
db_uuid
|
db_uuid
|
||||||
)));
|
)));
|
||||||
}
|
|
||||||
|
|
||||||
// Ensure that "database release" works with "database restore"
|
|
||||||
#[tokio::test]
|
|
||||||
async fn release_restore_database() {
|
|
||||||
let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
|
|
||||||
let addr = server_fixture.grpc_base();
|
|
||||||
let db_name = rand_name();
|
|
||||||
let db = &db_name;
|
|
||||||
|
|
||||||
|
// Claiming a database with a valid but unknown UUID is an error
|
||||||
|
let unknown_uuid = Uuid::new_v4();
|
||||||
|
dbg!(unknown_uuid);
|
||||||
Command::cargo_bin("influxdb_iox")
|
Command::cargo_bin("influxdb_iox")
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.arg("database")
|
.arg("database")
|
||||||
.arg("create")
|
.arg("claim")
|
||||||
.arg(db)
|
.arg(unknown_uuid.to_string())
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains("Created"));
|
|
||||||
|
|
||||||
// Release the database, returns the UUID
|
|
||||||
let stdout = String::from_utf8(
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("release")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!(
|
|
||||||
"Released database {}",
|
|
||||||
db
|
|
||||||
)))
|
|
||||||
.get_output()
|
|
||||||
.stdout
|
|
||||||
.clone(),
|
|
||||||
)
|
|
||||||
.unwrap();
|
|
||||||
let db_uuid = stdout.lines().last().unwrap().trim();
|
|
||||||
|
|
||||||
// Creating a new database with the same name works
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("create")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains("Created"));
|
|
||||||
|
|
||||||
// Restoring the 1st database is an error because the new, currently active database has the
|
|
||||||
// same name
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("restore")
|
|
||||||
.arg(db_uuid)
|
|
||||||
.arg("--host")
|
.arg("--host")
|
||||||
.arg(addr)
|
.arg(addr)
|
||||||
.assert()
|
.assert()
|
||||||
.failure()
|
.failure()
|
||||||
.stderr(predicate::str::contains(format!(
|
.stderr(predicate::str::contains(format!(
|
||||||
"A database with the name `{}` already exists",
|
"Could not find a database with UUID `{}`",
|
||||||
db
|
unknown_uuid
|
||||||
)));
|
)));
|
||||||
|
|
||||||
// Release the 2nd database
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("release")
|
|
||||||
.arg(db)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!(
|
|
||||||
"Released database {}",
|
|
||||||
db
|
|
||||||
)));
|
|
||||||
|
|
||||||
// Restore the 1st database
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("restore")
|
|
||||||
.arg(db_uuid)
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(format!(
|
|
||||||
"Restored database {}",
|
|
||||||
db_uuid
|
|
||||||
)));
|
|
||||||
|
|
||||||
// The 1st database is back in the active list
|
|
||||||
Command::cargo_bin("influxdb_iox")
|
|
||||||
.unwrap()
|
|
||||||
.arg("database")
|
|
||||||
.arg("list")
|
|
||||||
.arg("--host")
|
|
||||||
.arg(addr)
|
|
||||||
.assert()
|
|
||||||
.success()
|
|
||||||
.stdout(predicate::str::contains(db));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|
|
@ -126,26 +126,6 @@ pub enum GetDatabaseError {
|
||||||
ServerError(tonic::Status),
|
ServerError(tonic::Status),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Errors returned by Client::delete_database
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum DeleteDatabaseError {
|
|
||||||
/// Database not found
|
|
||||||
#[error("Database not found")]
|
|
||||||
DatabaseNotFound,
|
|
||||||
|
|
||||||
/// Server indicated that it is not (yet) available
|
|
||||||
#[error("Server unavailable: {}", .0.message())]
|
|
||||||
Unavailable(tonic::Status),
|
|
||||||
|
|
||||||
/// Server ID is not set
|
|
||||||
#[error("Server ID not set")]
|
|
||||||
NoServerId,
|
|
||||||
|
|
||||||
/// Client received an unexpected error from the server
|
|
||||||
#[error("Unexpected server error: {}: {}", .0.code(), .0.message())]
|
|
||||||
ServerError(tonic::Status),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Errors returned by Client::release_database
|
/// Errors returned by Client::release_database
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum ReleaseDatabaseError {
|
pub enum ReleaseDatabaseError {
|
||||||
|
@ -173,29 +153,6 @@ pub enum ReleaseDatabaseError {
|
||||||
ServerError(tonic::Status),
|
ServerError(tonic::Status),
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Errors returned by Client::restore_database
|
|
||||||
#[derive(Debug, Error)]
|
|
||||||
pub enum RestoreDatabaseError {
|
|
||||||
/// Database not found
|
|
||||||
#[error("Could not find a database with UUID `{}`", .uuid)]
|
|
||||||
DatabaseNotFound {
|
|
||||||
/// The UUID requested
|
|
||||||
uuid: Uuid,
|
|
||||||
},
|
|
||||||
|
|
||||||
/// Server indicated that it is not (yet) available
|
|
||||||
#[error("Server unavailable: {}", .0.message())]
|
|
||||||
Unavailable(tonic::Status),
|
|
||||||
|
|
||||||
/// Server ID is not set
|
|
||||||
#[error("Server ID not set")]
|
|
||||||
NoServerId,
|
|
||||||
|
|
||||||
/// Client received an unexpected error from the server
|
|
||||||
#[error("Unexpected server error: {}: {}", .0.code(), .0.message())]
|
|
||||||
ServerError(tonic::Status),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Errors returned by Client::claim_database
|
/// Errors returned by Client::claim_database
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum ClaimDatabaseError {
|
pub enum ClaimDatabaseError {
|
||||||
|
@ -716,37 +673,6 @@ impl Client {
|
||||||
Ok(rules)
|
Ok(rules)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Delete database
|
|
||||||
pub async fn delete_database(
|
|
||||||
&mut self,
|
|
||||||
db_name: impl Into<String> + Send,
|
|
||||||
) -> Result<Uuid, DeleteDatabaseError> {
|
|
||||||
let response = self
|
|
||||||
.inner
|
|
||||||
.delete_database(DeleteDatabaseRequest {
|
|
||||||
db_name: db_name.into(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|status| match status.code() {
|
|
||||||
tonic::Code::NotFound => DeleteDatabaseError::DatabaseNotFound,
|
|
||||||
tonic::Code::FailedPrecondition => DeleteDatabaseError::NoServerId,
|
|
||||||
tonic::Code::Unavailable => DeleteDatabaseError::Unavailable(status),
|
|
||||||
_ => DeleteDatabaseError::ServerError(status),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let server_uuid = response.into_inner().uuid;
|
|
||||||
let uuid = Uuid::from_slice(&server_uuid)
|
|
||||||
.map_err(|e| {
|
|
||||||
format!(
|
|
||||||
"Could not create UUID from server value {:?}: {}",
|
|
||||||
server_uuid, e
|
|
||||||
)
|
|
||||||
})
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
Ok(uuid)
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Release database
|
/// Release database
|
||||||
pub async fn release_database(
|
pub async fn release_database(
|
||||||
&mut self,
|
&mut self,
|
||||||
|
@ -781,23 +707,6 @@ impl Client {
|
||||||
Ok(uuid)
|
Ok(uuid)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Restore database
|
|
||||||
pub async fn restore_database(&mut self, uuid: Uuid) -> Result<(), RestoreDatabaseError> {
|
|
||||||
self.inner
|
|
||||||
.restore_database(RestoreDatabaseRequest {
|
|
||||||
uuid: uuid.as_bytes().to_vec(),
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.map_err(|status| match status.code() {
|
|
||||||
tonic::Code::NotFound => RestoreDatabaseError::DatabaseNotFound { uuid },
|
|
||||||
tonic::Code::FailedPrecondition => RestoreDatabaseError::NoServerId,
|
|
||||||
tonic::Code::Unavailable => RestoreDatabaseError::Unavailable(status),
|
|
||||||
_ => RestoreDatabaseError::ServerError(status),
|
|
||||||
})?;
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Claim database
|
/// Claim database
|
||||||
pub async fn claim_database(&mut self, uuid: Uuid) -> Result<String, ClaimDatabaseError> {
|
pub async fn claim_database(&mut self, uuid: Uuid) -> Result<String, ClaimDatabaseError> {
|
||||||
let uuid_bytes = uuid.as_bytes().to_vec();
|
let uuid_bytes = uuid.as_bytes().to_vec();
|
||||||
|
|
|
@ -41,20 +41,8 @@ pub enum IoxObjectStoreError {
|
||||||
#[snafu(display("Cannot create database with UUID `{}`; it already exists", uuid))]
|
#[snafu(display("Cannot create database with UUID `{}`; it already exists", uuid))]
|
||||||
DatabaseAlreadyExists { uuid: Uuid },
|
DatabaseAlreadyExists { uuid: Uuid },
|
||||||
|
|
||||||
#[snafu(display(
|
|
||||||
"Cannot restore; there is already an active database with UUID `{}`",
|
|
||||||
uuid
|
|
||||||
))]
|
|
||||||
DatabaseAlreadyActive { uuid: Uuid },
|
|
||||||
|
|
||||||
#[snafu(display("No rules found to load at {}", root_path))]
|
#[snafu(display("No rules found to load at {}", root_path))]
|
||||||
NoRulesFound { root_path: RootPath },
|
NoRulesFound { root_path: RootPath },
|
||||||
|
|
||||||
#[snafu(display("Could not restore database with UUID `{}`: {}", uuid, source))]
|
|
||||||
RestoreFailed {
|
|
||||||
uuid: Uuid,
|
|
||||||
source: object_store::Error,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Handles persistence of data for a particular database. Writes within its directory/prefix.
|
/// Handles persistence of data for a particular database. Writes within its directory/prefix.
|
||||||
|
|
|
@ -81,18 +81,6 @@ pub enum Error {
|
||||||
#[snafu(display("cannot persisted updated rules: {}", source))]
|
#[snafu(display("cannot persisted updated rules: {}", source))]
|
||||||
CannotPersistUpdatedRules { source: crate::rules::Error },
|
CannotPersistUpdatedRules { source: crate::rules::Error },
|
||||||
|
|
||||||
#[snafu(display("cannot mark database {} deleted: {}", db_name, source))]
|
|
||||||
CannotMarkDatabaseDeleted {
|
|
||||||
db_name: String,
|
|
||||||
source: object_store::Error,
|
|
||||||
},
|
|
||||||
|
|
||||||
#[snafu(display(
|
|
||||||
"cannot delete database named {} that has already been marked as deleted",
|
|
||||||
db_name
|
|
||||||
))]
|
|
||||||
CannotDeleteInactiveDatabase { db_name: String },
|
|
||||||
|
|
||||||
#[snafu(display(
|
#[snafu(display(
|
||||||
"cannot release database named {} that has already been released",
|
"cannot release database named {} that has already been released",
|
||||||
db_name
|
db_name
|
||||||
|
|
|
@ -158,24 +158,15 @@ pub enum Error {
|
||||||
#[snafu(display("cannot get database name from rules: {}", source))]
|
#[snafu(display("cannot get database name from rules: {}", source))]
|
||||||
CouldNotGetDatabaseNameFromRules { source: DatabaseNameFromRulesError },
|
CouldNotGetDatabaseNameFromRules { source: DatabaseNameFromRulesError },
|
||||||
|
|
||||||
#[snafu(display("{}", source))]
|
|
||||||
CannotMarkDatabaseDeleted { source: crate::database::Error },
|
|
||||||
|
|
||||||
#[snafu(display("{}", source))]
|
#[snafu(display("{}", source))]
|
||||||
CannotReleaseDatabase { source: crate::database::Error },
|
CannotReleaseDatabase { source: crate::database::Error },
|
||||||
|
|
||||||
#[snafu(display("{}", source))]
|
|
||||||
CannotRestoreDatabase { source: crate::database::InitError },
|
|
||||||
|
|
||||||
#[snafu(display("{}", source))]
|
#[snafu(display("{}", source))]
|
||||||
CannotClaimDatabase { source: crate::database::InitError },
|
CannotClaimDatabase { source: crate::database::InitError },
|
||||||
|
|
||||||
#[snafu(display("A database with the name `{}` already exists", db_name))]
|
#[snafu(display("A database with the name `{}` already exists", db_name))]
|
||||||
DatabaseAlreadyExists { db_name: String },
|
DatabaseAlreadyExists { db_name: String },
|
||||||
|
|
||||||
#[snafu(display("The database with UUID `{}` named `{}` is already active", uuid, name))]
|
|
||||||
DatabaseAlreadyActive { name: String, uuid: Uuid },
|
|
||||||
|
|
||||||
#[snafu(display("The database with UUID `{}` is already owned by this server", uuid))]
|
#[snafu(display("The database with UUID `{}` is already owned by this server", uuid))]
|
||||||
DatabaseAlreadyOwnedByThisServer { uuid: Uuid },
|
DatabaseAlreadyOwnedByThisServer { uuid: Uuid },
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue