feat: Move server config paths beneath 'nodes' (#3144)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>

influxdb_iox/tests/end_to_end_cases/management_api.rs

@@ -1349,7 +1349,7 @@ async fn test_get_server_status_db_error() {
     // create valid owner info but malformed DB rules that will put DB in an error state
     let my_db_uuid = Uuid::new_v4();
     let mut path = server_fixture.dir().to_path_buf();
-    path.push("42");
+    path.push("dbs");
     path.push(my_db_uuid.to_string());
     std::fs::create_dir_all(path.clone()).unwrap();
     let mut owner_info_path = path.clone();
@@ -1360,11 +1360,13 @@ async fn test_get_server_status_db_error() {
 
     // create the server config listing the ownership of this database
     let mut path = server_fixture.dir().to_path_buf();
+    path.push("nodes");
     path.push("42");
+    std::fs::create_dir_all(path.clone()).unwrap();
     path.push("config.pb");
 
     let data = ServerConfig {
-        databases: vec![(String::from("my_db"), format!("42/{}", my_db_uuid))]
+        databases: vec![(String::from("my_db"), format!("dbs/{}", my_db_uuid))]
             .into_iter()
             .collect(),
     };

iox_object_store/src/lib.rs

@@ -59,9 +59,23 @@ pub struct IoxObjectStore {
 impl IoxObjectStore {
     /// Get the data for the server config to determine the names and locations of the databases
     /// that this server owns.
+    ///
+    /// TEMPORARY: Server config used to be at the top level instead of beneath `/nodes/`. Until
+    /// all deployments have transitioned, check both locations before reporting that the server
+    /// config is not found.
     pub async fn get_server_config_file(inner: &ObjectStore, server_id: ServerId) -> Result<Bytes> {
         let path = paths::server_config_path(inner, server_id);
-        let mut stream = inner.get(&path).await?;
+        let mut stream = match inner.get(&path).await {
+            Err(object_store::Error::NotFound { .. }) => {
+                use object_store::path::ObjectStorePath;
+                let mut legacy_path = inner.new_path();
+                legacy_path.push_dir(server_id.to_string());
+                legacy_path.set_file_name(paths::SERVER_CONFIG_FILE_NAME);
+
+                inner.get(&legacy_path).await
+            }
+            other => other,
+        }?;
         let mut bytes = BytesMut::new();
 
         while let Some(buf) = stream.next().await {

iox_object_store/src/paths.rs

@@ -15,13 +15,15 @@ pub mod transaction_file;
 use transaction_file::TransactionFilePath;
 
 pub(crate) const ALL_DATABASES_DIRECTORY: &str = "dbs";
-const SERVER_CONFIG_FILE_NAME: &str = "config.pb";
+const ALL_SERVERS_DIRECTORY: &str = "nodes";
+pub(crate) const SERVER_CONFIG_FILE_NAME: &str = "config.pb";
 const DATABASE_OWNER_FILE_NAME: &str = "owner.pb";
 
 /// The path to the server file containing the list of databases this server owns.
 // TODO: this is in the process of replacing all_databases_path for the floating databases design
 pub(crate) fn server_config_path(object_store: &ObjectStore, server_id: ServerId) -> Path {
     let mut path = object_store.new_path();
+    path.push_dir(ALL_SERVERS_DIRECTORY);
     path.push_dir(server_id.to_string());
     path.set_file_name(SERVER_CONFIG_FILE_NAME);
     path

server/src/lib.rs

@@ -1875,6 +1875,95 @@ mod tests {
         new_loc_db.wait_for_init().await.unwrap();
     }
 
+    #[tokio::test]
+    async fn old_server_config_object_store_path() {
+        let application = make_application();
+        let server_id = ServerId::try_from(1).unwrap();
+        let object_store = application.object_store();
+
+        // Server config used to be stored under /[server id]/config.pb. Construct a config in that
+        // old location that points to a database
+        let mut old_server_config_path = object_store.new_path();
+        old_server_config_path.push_dir(&server_id.to_string());
+        old_server_config_path.set_file_name("config.pb");
+
+        // Create database rules and database owner info for a database in object storage
+        let db_uuid = Uuid::new_v4();
+        let db_name = DatabaseName::new("mydb").unwrap();
+        let db_rules = DatabaseRules::new(db_name.clone());
+
+        let mut db_path = object_store.new_path();
+        db_path.push_dir("dbs");
+        db_path.push_dir(db_uuid.to_string());
+        let mut db_rules_path = db_path.clone();
+        db_rules_path.set_file_name("rules.pb");
+
+        let persisted_database_rules = management::v1::PersistedDatabaseRules {
+            uuid: db_uuid.as_bytes().to_vec(),
+            rules: Some(db_rules.into()),
+        };
+        let mut encoded_rules = bytes::BytesMut::new();
+        generated_types::database_rules::encode_persisted_database_rules(
+            &persisted_database_rules,
+            &mut encoded_rules,
+        )
+        .unwrap();
+        let encoded_rules = encoded_rules.freeze();
+        object_store
+            .put(&db_rules_path, encoded_rules)
+            .await
+            .unwrap();
+
+        let mut db_owner_info_path = db_path.clone();
+        db_owner_info_path.set_file_name("owner.pb");
+        let owner_info = management::v1::OwnerInfo {
+            id: server_id.get_u32(),
+            location: old_server_config_path.to_string(),
+            transactions: vec![],
+        };
+        let mut encoded_owner_info = bytes::BytesMut::new();
+        generated_types::server_config::encode_database_owner_info(
+            &owner_info,
+            &mut encoded_owner_info,
+        )
+        .unwrap();
+        let encoded_owner_info = encoded_owner_info.freeze();
+        object_store
+            .put(&db_owner_info_path, encoded_owner_info)
+            .await
+            .unwrap();
+
+        let config = management::v1::ServerConfig {
+            databases: [(db_name.to_string(), db_path.to_raw())]
+                .into_iter()
+                .collect(),
+        };
+        let mut encoded_server_config = bytes::BytesMut::new();
+        generated_types::server_config::encode_persisted_server_config(
+            &config,
+            &mut encoded_server_config,
+        )
+        .unwrap();
+        let encoded_server_config = encoded_server_config.freeze();
+        object_store
+            .put(&old_server_config_path, encoded_server_config)
+            .await
+            .unwrap();
+
+        // Start up server
+        let server = make_server(Arc::clone(&application));
+        server.set_id(server_id).unwrap();
+        server.wait_for_init().await.unwrap();
+
+        // Database should init
+        let database = server.database(&db_name).unwrap();
+        database.wait_for_init().await.unwrap();
+
+        // Server config should be transitioned to the new location
+        let config = server_config(application.object_store(), server_id).await;
+        assert_config_contents(&config, &[(&db_name, format!("dbs/{}/", db_uuid))]);
+    }
+
     #[tokio::test]
     async fn db_names_sorted() {
         let server = make_server(make_application());
@@ -2232,7 +2321,7 @@ mod tests {
         let baz_iox_object_store = baz.iox_object_store().unwrap();
         let owner_info = management::v1::OwnerInfo {
             id: 2,
-            location: "2/config.pb".to_string(),
+            location: "nodes/2/config.pb".to_string(),
             transactions: vec![],
         };
         let mut encoded = bytes::BytesMut::new();