From 25d55cd08a6082e1448ca4657dab708eb2027f13 Mon Sep 17 00:00:00 2001 From: "Carol (Nichols || Goulding)" <193874+carols10cents@users.noreply.github.com> Date: Fri, 19 Nov 2021 04:54:32 -0500 Subject: [PATCH] feat: Move server config paths beneath 'nodes' (#3144) Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- .../tests/end_to_end_cases/management_api.rs | 6 +- iox_object_store/src/lib.rs | 16 +++- iox_object_store/src/paths.rs | 4 +- server/src/lib.rs | 91 ++++++++++++++++++- 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/influxdb_iox/tests/end_to_end_cases/management_api.rs b/influxdb_iox/tests/end_to_end_cases/management_api.rs index 4754983129..53e7052499 100644 --- a/influxdb_iox/tests/end_to_end_cases/management_api.rs +++ b/influxdb_iox/tests/end_to_end_cases/management_api.rs @@ -1349,7 +1349,7 @@ async fn test_get_server_status_db_error() { // create valid owner info but malformed DB rules that will put DB in an error state let my_db_uuid = Uuid::new_v4(); let mut path = server_fixture.dir().to_path_buf(); - path.push("42"); + path.push("dbs"); path.push(my_db_uuid.to_string()); std::fs::create_dir_all(path.clone()).unwrap(); let mut owner_info_path = path.clone(); @@ -1360,11 +1360,13 @@ async fn test_get_server_status_db_error() { // create the server config listing the ownership of this database let mut path = server_fixture.dir().to_path_buf(); + path.push("nodes"); path.push("42"); + std::fs::create_dir_all(path.clone()).unwrap(); path.push("config.pb"); let data = ServerConfig { - databases: vec![(String::from("my_db"), format!("42/{}", my_db_uuid))] + databases: vec![(String::from("my_db"), format!("dbs/{}", my_db_uuid))] .into_iter() .collect(), }; diff --git a/iox_object_store/src/lib.rs b/iox_object_store/src/lib.rs index 7c5b855912..db735f08e2 100644 --- a/iox_object_store/src/lib.rs +++ b/iox_object_store/src/lib.rs @@ -59,9 +59,23 @@ pub struct IoxObjectStore { impl IoxObjectStore { /// Get the data for the server config to determine the names and locations of the databases /// that this server owns. + /// + /// TEMPORARY: Server config used to be at the top level instead of beneath `/nodes/`. Until + /// all deployments have transitioned, check both locations before reporting that the server + /// config is not found. pub async fn get_server_config_file(inner: &ObjectStore, server_id: ServerId) -> Result { let path = paths::server_config_path(inner, server_id); - let mut stream = inner.get(&path).await?; + let mut stream = match inner.get(&path).await { + Err(object_store::Error::NotFound { .. }) => { + use object_store::path::ObjectStorePath; + let mut legacy_path = inner.new_path(); + legacy_path.push_dir(server_id.to_string()); + legacy_path.set_file_name(paths::SERVER_CONFIG_FILE_NAME); + + inner.get(&legacy_path).await + } + other => other, + }?; let mut bytes = BytesMut::new(); while let Some(buf) = stream.next().await { diff --git a/iox_object_store/src/paths.rs b/iox_object_store/src/paths.rs index 5b5a992dc6..274ebb1ba6 100644 --- a/iox_object_store/src/paths.rs +++ b/iox_object_store/src/paths.rs @@ -15,13 +15,15 @@ pub mod transaction_file; use transaction_file::TransactionFilePath; pub(crate) const ALL_DATABASES_DIRECTORY: &str = "dbs"; -const SERVER_CONFIG_FILE_NAME: &str = "config.pb"; +const ALL_SERVERS_DIRECTORY: &str = "nodes"; +pub(crate) const SERVER_CONFIG_FILE_NAME: &str = "config.pb"; const DATABASE_OWNER_FILE_NAME: &str = "owner.pb"; /// The path to the server file containing the list of databases this server owns. // TODO: this is in the process of replacing all_databases_path for the floating databases design pub(crate) fn server_config_path(object_store: &ObjectStore, server_id: ServerId) -> Path { let mut path = object_store.new_path(); + path.push_dir(ALL_SERVERS_DIRECTORY); path.push_dir(server_id.to_string()); path.set_file_name(SERVER_CONFIG_FILE_NAME); path diff --git a/server/src/lib.rs b/server/src/lib.rs index 960799dcfa..ddfd2f47c0 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -1875,6 +1875,95 @@ mod tests { new_loc_db.wait_for_init().await.unwrap(); } + #[tokio::test] + async fn old_server_config_object_store_path() { + let application = make_application(); + let server_id = ServerId::try_from(1).unwrap(); + let object_store = application.object_store(); + + // Server config used to be stored under /[server id]/config.pb. Construct a config in that + // old location that points to a database + let mut old_server_config_path = object_store.new_path(); + old_server_config_path.push_dir(&server_id.to_string()); + old_server_config_path.set_file_name("config.pb"); + + // Create database rules and database owner info for a database in object storage + let db_uuid = Uuid::new_v4(); + let db_name = DatabaseName::new("mydb").unwrap(); + let db_rules = DatabaseRules::new(db_name.clone()); + + let mut db_path = object_store.new_path(); + db_path.push_dir("dbs"); + db_path.push_dir(db_uuid.to_string()); + let mut db_rules_path = db_path.clone(); + db_rules_path.set_file_name("rules.pb"); + + let persisted_database_rules = management::v1::PersistedDatabaseRules { + uuid: db_uuid.as_bytes().to_vec(), + rules: Some(db_rules.into()), + }; + let mut encoded_rules = bytes::BytesMut::new(); + generated_types::database_rules::encode_persisted_database_rules( + &persisted_database_rules, + &mut encoded_rules, + ) + .unwrap(); + let encoded_rules = encoded_rules.freeze(); + object_store + .put(&db_rules_path, encoded_rules) + .await + .unwrap(); + + let mut db_owner_info_path = db_path.clone(); + db_owner_info_path.set_file_name("owner.pb"); + let owner_info = management::v1::OwnerInfo { + id: server_id.get_u32(), + location: old_server_config_path.to_string(), + transactions: vec![], + }; + let mut encoded_owner_info = bytes::BytesMut::new(); + generated_types::server_config::encode_database_owner_info( + &owner_info, + &mut encoded_owner_info, + ) + .unwrap(); + let encoded_owner_info = encoded_owner_info.freeze(); + object_store + .put(&db_owner_info_path, encoded_owner_info) + .await + .unwrap(); + + let config = management::v1::ServerConfig { + databases: [(db_name.to_string(), db_path.to_raw())] + .into_iter() + .collect(), + }; + let mut encoded_server_config = bytes::BytesMut::new(); + generated_types::server_config::encode_persisted_server_config( + &config, + &mut encoded_server_config, + ) + .unwrap(); + let encoded_server_config = encoded_server_config.freeze(); + object_store + .put(&old_server_config_path, encoded_server_config) + .await + .unwrap(); + + // Start up server + let server = make_server(Arc::clone(&application)); + server.set_id(server_id).unwrap(); + server.wait_for_init().await.unwrap(); + + // Database should init + let database = server.database(&db_name).unwrap(); + database.wait_for_init().await.unwrap(); + + // Server config should be transitioned to the new location + let config = server_config(application.object_store(), server_id).await; + assert_config_contents(&config, &[(&db_name, format!("dbs/{}/", db_uuid))]); + } + #[tokio::test] async fn db_names_sorted() { let server = make_server(make_application()); @@ -2232,7 +2321,7 @@ mod tests { let baz_iox_object_store = baz.iox_object_store().unwrap(); let owner_info = management::v1::OwnerInfo { id: 2, - location: "2/config.pb".to_string(), + location: "nodes/2/config.pb".to_string(), transactions: vec![], }; let mut encoded = bytes::BytesMut::new();