Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #2949 from influxdata/cn/remove-list-deleted-databases 

fix: Remove the ability to list deleted databases
pull/24376/head
kodiakhq[bot] 2021-10-22 19:43:09 +00:00 committed by GitHub
parent 38f849d2fb 9b70001aa4
commit 6ab8bc6ec0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 13 additions and 280 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
generated_types/protos/influxdata/iox/management/v1/service.proto
View File

@ -37,10 +37,7 @@ service ManagementService {
rpc RestoreDatabase(RestoreDatabaseRequest) returns (RestoreDatabaseResponse);
// List deleted databases and their metadata.
rpc ListDeletedDatabases(ListDeletedDatabasesRequest) returns (ListDeletedDatabasesResponse);
// List all databases and their metadata.
// List databases with their metadata.
rpc ListDetailedDatabases(ListDetailedDatabasesRequest) returns (ListDetailedDatabasesResponse);
// List chunks available on this database
@ -189,12 +186,6 @@ message RestoreDatabaseRequest {
message RestoreDatabaseResponse {}
message ListDeletedDatabasesRequest {}
message ListDeletedDatabasesResponse {
repeated DetailedDatabase deleted_databases = 1;
}
message ListDetailedDatabasesRequest {}
message ListDetailedDatabasesResponse {

17
influxdb_iox_client/src/client/management.rs
View File

@ -602,22 +602,7 @@ impl Client {
Ok(names)
}
/// List deleted databases and metadata
pub async fn list_deleted_databases(
&mut self,
) -> Result<Vec<DetailedDatabase>, ListDatabaseError> {
let response = self
.inner
.list_deleted_databases(ListDeletedDatabasesRequest {})
.await
.map_err(|status| match status.code() {
tonic::Code::Unavailable => ListDatabaseError::Unavailable(status),
_ => ListDatabaseError::ServerError(status),
})?;
Ok(response.into_inner().deleted_databases)
}
/// List all databases and detailed metadata
/// List databases and detailed metadata
pub async fn list_detailed_databases(
&mut self,
) -> Result<Vec<DetailedDatabase>, ListDatabaseError> {

130
iox_object_store/src/lib.rs
View File

@ -184,33 +184,7 @@ impl IoxObjectStore {
.collect())
}
/// List databases marked as deleted in in object storage along with their generation IDs and
/// when they were deleted. Enables a user to choose a generation for a database that they
/// would want to restore or would want to delete permanently.
pub async fn list_deleted_databases(
inner: &ObjectStore,
server_id: ServerId,
) -> Result<Vec<DetailedDatabase>> {
Ok(Self::list_all_databases(inner, server_id)
.await?
.into_iter()
.flat_map(|(name, generations)| {
let name = Arc::new(name);
generations.into_iter().filter_map(move |gen| {
let name = Arc::clone(&name);
gen.deleted_at.map(|_| DetailedDatabase {
name: (*name).clone(),
generation_id: gen.id,
deleted_at: gen.deleted_at,
})
})
})
.collect())
}
/// List all databases in in object storage along with their generation IDs and if/when they
/// were deleted. Useful for visibility into object storage and finding databases to restore or
/// permanently delete.
/// List this server's databases in object storage along with their generation IDs.
pub async fn list_detailed_databases(
inner: &ObjectStore,
server_id: ServerId,
@ -1298,93 +1272,6 @@ mod tests {
);
}
#[tokio::test]
async fn list_deleted_databases_metadata() {
let object_store = make_object_store();
let server_id = make_server_id();
// Create a normal database, will NOT be in the list of deleted databases
let db_normal = DatabaseName::new("db_normal").unwrap();
create_database(Arc::clone(&object_store), server_id, &db_normal).await;
// Create a database, then delete it - will be in the list once
let db_deleted = DatabaseName::new("db_deleted").unwrap();
let db_deleted_iox_store =
create_database(Arc::clone(&object_store), server_id, &db_deleted).await;
delete_database(&db_deleted_iox_store).await;
// Create, delete, create - will be in the list once
let db_reincarnated = DatabaseName::new("db_reincarnated").unwrap();
let db_reincarnated_iox_store =
create_database(Arc::clone(&object_store), server_id, &db_reincarnated).await;
delete_database(&db_reincarnated_iox_store).await;
create_database(Arc::clone(&object_store), server_id, &db_reincarnated).await;
// Create, delete, create, delete - will be in the list twice
let db_deleted_twice = DatabaseName::new("db_deleted_twice").unwrap();
let db_deleted_twice_iox_store =
create_database(Arc::clone(&object_store), server_id, &db_deleted_twice).await;
delete_database(&db_deleted_twice_iox_store).await;
let db_deleted_twice_iox_store =
create_database(Arc::clone(&object_store), server_id, &db_deleted_twice).await;
delete_database(&db_deleted_twice_iox_store).await;
// Put a file in a directory that looks like a database directory but has no rules,
// won't be in the list because there's no tombstone file
let not_a_db = DatabaseName::new("not_a_db").unwrap();
let mut not_rules_path = object_store.new_path();
not_rules_path.push_all_dirs(&[&server_id.to_string(), not_a_db.as_str(), "0"]);
not_rules_path.set_file_name("not_rules.txt");
object_store
.put(&not_rules_path, Bytes::new())
.await
.unwrap();
// Put a file in a directory that's an invalid database name - won't be in the list
let invalid_db_name = ("a".repeat(65)).to_string();
let mut invalid_db_name_rules_path = object_store.new_path();
invalid_db_name_rules_path.push_all_dirs(&[&server_id.to_string(), &invalid_db_name, "0"]);
invalid_db_name_rules_path.set_file_name("rules.pb");
object_store
.put(&invalid_db_name_rules_path, Bytes::new())
.await
.unwrap();
// Put a file in a directory that looks like a database name, but doesn't look like a
// generation directory - won't be in the list
let no_generations = DatabaseName::new("no_generations").unwrap();
let mut no_generations_path = object_store.new_path();
no_generations_path.push_all_dirs(&[
&server_id.to_string(),
no_generations.as_str(),
"not-a-generation",
]);
no_generations_path.set_file_name("not_rules.txt");
object_store
.put(&no_generations_path, Bytes::new())
.await
.unwrap();
let mut deleted_dbs = IoxObjectStore::list_deleted_databases(&object_store, server_id)
.await
.unwrap();
deleted_dbs.sort_by_key(|d| (d.name.clone(), d.generation_id));
assert_eq!(deleted_dbs.len(), 4);
assert_eq!(deleted_dbs[0].name, db_deleted);
assert_eq!(deleted_dbs[0].generation_id.inner, 0);
assert_eq!(deleted_dbs[1].name, db_deleted_twice);
assert_eq!(deleted_dbs[1].generation_id.inner, 0);
assert_eq!(deleted_dbs[2].name, db_deleted_twice);
assert_eq!(deleted_dbs[2].generation_id.inner, 1);
assert_eq!(deleted_dbs[3].name, db_reincarnated);
assert_eq!(deleted_dbs[3].generation_id.inner, 0);
}
async fn restore_database(
object_store: Arc<ObjectStore>,
server_id: ServerId,
@ -1417,11 +1304,12 @@ mod tests {
delete_database(&db_iox_store).await;
// Get one generation ID from the list of deleted databases
let deleted_dbs = IoxObjectStore::list_deleted_databases(&object_store, server_id)
let listed_dbs = IoxObjectStore::list_detailed_databases(&object_store, server_id)
.await
.unwrap();
assert_eq!(deleted_dbs.len(), 2);
let deleted_db = deleted_dbs.iter().find(|d| d.name == db).unwrap();
assert_eq!(listed_dbs.len(), 2);
let deleted_generations: Vec<_> = listed_dbs.iter().filter(|d| d.name == db).collect();
let deleted_db = deleted_generations[0];
// Restore the generation
restore_database(
@ -1439,14 +1327,8 @@ mod tests {
.unwrap();
assert_eq!(all_dbs.len(), 1);
// The other deleted generation should be the only item in the deleted list
let deleted_dbs = IoxObjectStore::list_deleted_databases(&object_store, server_id)
.await
.unwrap();
assert_eq!(deleted_dbs.len(), 1);
// Try to restore the other deleted database
let deleted_db = deleted_dbs.iter().find(|d| d.name == db).unwrap();
let deleted_db = deleted_generations[1];
let err = restore_database(
Arc::clone(&object_store),
server_id,

18
server/src/lib.rs
View File

@ -793,23 +793,7 @@ where
Ok(())
}
/// List deleted databases in object storage.
pub async fn list_deleted_databases(&self) -> Result<Vec<DetailedDatabase>> {
let server_id = {
let state = self.shared.state.read();
let initialized = state.initialized()?;
initialized.server_id
};
Ok(IoxObjectStore::list_deleted_databases(
self.shared.application.object_store(),
server_id,
)
.await
.context(ListDeletedDatabases)?)
}
/// List all databases, active and deleted, in object storage, including their generation IDs.
/// List this server's databases in object storage, including their generation IDs.
pub async fn list_detailed_databases(&self) -> Result<Vec<DetailedDatabase>> {
let server_id = {
let state = self.shared.state.read();

16
src/commands/database.rs
View File

@ -139,12 +139,8 @@ struct Create {
/// Get list of databases
#[derive(Debug, StructOpt)]
struct List {
/// Whether to list databases marked as deleted instead, to restore or permanently delete.
#[structopt(long)]
deleted: bool,
/// Whether to list detailed information, including generation IDs, about all databases,
/// whether they are active or marked as deleted.
/// Whether to list detailed information about the databases, such as generation IDs along
/// with their names.
#[structopt(long)]
detailed: bool,
}
@ -264,12 +260,8 @@ pub async fn command(connection: Connection, config: Config) -> Result<()> {
}
Command::List(list) => {
let mut client = management::Client::new(connection);
if list.deleted || list.detailed {
let databases = if list.deleted {
client.list_deleted_databases().await?
} else {
client.list_detailed_databases().await?
};
if list.detailed {
let databases = client.list_detailed_databases().await?;
let mut table = Table::new();
table.load_preset("||--+-++| ++++++");

18
src/influxdb_ioxd/rpc/management.rs
View File

@ -199,24 +199,6 @@ where
Ok(Response::new(RestoreDatabaseResponse {}))
}
async fn list_deleted_databases(
&self,
_: Request<ListDeletedDatabasesRequest>,
) -> Result<Response<ListDeletedDatabasesResponse>, Status> {
let deleted_databases = self
.server
.list_deleted_databases()
.await
.map_err(default_server_error_handler)?
.into_iter()
.map(Into::into)
.collect();
Ok(Response::new(ListDeletedDatabasesResponse {
deleted_databases,
}))
}
async fn list_detailed_databases(
&self,
_: Request<ListDetailedDatabasesRequest>,

23
tests/end_to_end_cases/management_api.rs
View File

@ -237,15 +237,6 @@ async fn test_list_databases() {
assert!(rules.lifecycle_rules.is_some());
}
// validate that neither database appears in the list of deleted databases
let deleted_databases = client
.list_deleted_databases()
.await
.expect("list deleted databases failed");
let names: Vec<_> = deleted_databases.into_iter().map(|db| db.db_name).collect();
assert!(!names.contains(&name1));
assert!(!names.contains(&name2));
// now fetch without defaults, and neither should have their rules filled in
let omit_defaults = true;
let databases: Vec<_> = client
@ -299,20 +290,6 @@ async fn test_list_databases() {
let names: Vec<_> = databases.iter().map(|rules| rules.name.clone()).collect();
assert!(!dbg!(&names).contains(&name1));
assert!(dbg!(&names).contains(&name2));
// The deleted database should be included in the list of deleted databases
let deleted_databases = client
.list_deleted_databases()
.await
.expect("list deleted databases failed");
assert!(
deleted_databases
.iter()
.any(|db| db.db_name == name1 && db.generation_id == 0),
"could not find expected database in {:?}",
deleted_databases
);
}
#[tokio::test]

60
tests/end_to_end_cases/management_cli.rs
View File

@ -232,18 +232,6 @@ async fn delete_database() {
.success()
.stdout(predicate::str::contains(db));
// Listing deleted databases does not include the newly created, active database
Command::cargo_bin("influxdb_iox")
.unwrap()
.arg("database")
.arg("list")
.arg("--deleted")
.arg("--host")
.arg(addr)
.assert()
.success()
.stdout(predicate::str::contains(db).not());
// Listing detailed database info does include the active database, along with its generation
Command::cargo_bin("influxdb_iox")
.unwrap()
@ -279,18 +267,6 @@ async fn delete_database() {
.success()
.stdout(predicate::str::contains(db).not());
// ... unless we ask to list deleted databases
Command::cargo_bin("influxdb_iox")
.unwrap()
.arg("database")
.arg("list")
.arg("--deleted")
.arg("--host")
.arg(addr)
.assert()
.success()
.stdout(deleted_db_match(db, 0));
// Listing detailed database info does include the deleted database
Command::cargo_bin("influxdb_iox")
.unwrap()
@ -340,18 +316,6 @@ async fn delete_database() {
.success()
.stdout(predicate::str::contains(db));
// And the one deleted database will be in the deleted list
Command::cargo_bin("influxdb_iox")
.unwrap()
.arg("database")
.arg("list")
.arg("--deleted")
.arg("--host")
.arg(addr)
.assert()
.success()
.stdout(deleted_db_match(db, 0));
// Listing detailed database info includes both active and deleted
Command::cargo_bin("influxdb_iox")
.unwrap()
@ -387,18 +351,6 @@ async fn delete_database() {
.success()
.stdout(predicate::str::contains(db).not());
// The 2 generations of the database should be in the deleted list
Command::cargo_bin("influxdb_iox")
.unwrap()
.arg("database")
.arg("list")
.arg("--deleted")
.arg("--host")
.arg(addr)
.assert()
.success()
.stdout(deleted_db_match(db, 0).and(deleted_db_match(db, 1)));
// Listing detailed database info includes both deleted generations
Command::cargo_bin("influxdb_iox")
.unwrap()
@ -438,18 +390,6 @@ async fn delete_database() {
.success()
.stdout(predicate::str::contains(db));
// Only generation 1 is in the deleted list
Command::cargo_bin("influxdb_iox")
.unwrap()
.arg("database")
.arg("list")
.arg("--deleted")
.arg("--host")
.arg(addr)
.assert()
.success()
.stdout(deleted_db_match(db, 0).not().and(deleted_db_match(db, 1)));
// Listing detailed database info includes both active and deleted
Command::cargo_bin("influxdb_iox")
.unwrap()