fix: Instead of using DATABASE_URL, use INFLUXDB_IOX_CATALOG_DSN and TEST_INFLUXDB_IOX_CATALOG_DSN

.circleci/config.yml

@@ -227,7 +227,7 @@ jobs:
       AZURE_USE_EMULATOR: "1"
       INFLUXDB_IOX_BUCKET: iox-test
       POSTGRES_USER: postgres
-      DATABASE_URL: "postgres://postgres@localhost/iox_shared"
+      TEST_INFLUXDB_IOX_CATALOG_DSN: "postgres://postgres@localhost/iox_shared"
     steps:
       - run:
           name: Setup localstack (AWS emulation)
@@ -247,8 +247,8 @@ jobs:
       - rust_components
       - cache_restore
       - run: cargo install sqlx-cli
-      - run: sqlx database create
-      - run: INFLUXDB_IOX_CATALOG_DSN=${DATABASE_URL} cargo run -- catalog setup
+      - run: DATABASE_URL=${TEST_INFLUXDB_IOX_CATALOG_DSN} sqlx database create
+      - run: INFLUXDB_IOX_CATALOG_DSN=${TEST_INFLUXDB_IOX_CATALOG_DSN} cargo run -- catalog setup
       - run:
           name: Cargo test
           command: cargo test --workspace --features=aws,azure,azure_test

iox_catalog/README.md

@@ -1,29 +1,31 @@
 # IOx Catalog
+
 This crate contains the code for the IOx Catalog. This includes the definitions of namespaces, their tables,
 the columns of those tables and their types, what Parquet files are in object storage and delete tombstones.
 There's also some configuration information that the overal distributed system uses for operation.
 
 To run this crate's tests you'll need Postgres installed and running locally. You'll also need to set the
-`DATABASE_URL` environment variable so that sqlx will be able to connect to your local DB. For example with
-user and password filled in:
+`INFLUXDB_IOX_CATALOG_DSN` environment variable so that sqlx will be able to connect to your local DB. For
+example with user and password filled in:
 
 ```
-DATABASE_URL=postgres://<postgres user>:<postgres password>@localhost/iox_shared
+INFLUXDB_IOX_CATALOG_DSN=postgres://<postgres user>:<postgres password>@localhost/iox_shared
 ```
 
-You can omit the host part if your postgres is running on the default unix domain socket (useful on macos because, by default,
-the config installed by `brew install postgres` doesn't listen to a TCP port): 
+You can omit the host part if your postgres is running on the default unix domain socket (useful on macos
+because, by default, the config installed by `brew install postgres` doesn't listen to a TCP port):
 
 ```
-DATABASE_URL=postgres:///iox_shared
+INFLUXDB_IOX_CATALOG_DSN=postgres:///iox_shared
 ```
 
 You'll then need to create the database. You can do this via the sqlx command line.
 
 ```
 cargo install sqlx-cli
-sqlx database create
-cargo run -q -- catalog setup --catalog-dsn "$DATABASE_URL"
+DATABASE_URL=<dsn> sqlx database create
+cargo run -q -- catalog setup
+cargo run -- catalog topic update iox-shared
 ```
 
 This will set up the database based on the files in `./migrations` in this crate. SQLx also creates a table
@@ -32,15 +34,38 @@ to keep track of which migrations have been run.
 NOTE: **do not** use `sqlx database setup`, because that will create the migration table in the wrong schema (namespace).
 Our `catalog setup` code will do that part by using the same sqlx migration module but with the right namespace setup.
 
+## Migrations
+
+If you need to create and run migrations to add, remove, or change the schema, you'll need the `sqlx-cli` tool. Install
+with `cargo install sqlx-cli` if you haven't already, then run `sqlx migrate --help` to see the commands relevant to
+migrations.
+
 ## Tests
 
 To run the Postgres integration tests, ensure the above setup is complete first.
 
-* Set `DATABASE_URL=<dsn>` env (see above)
-* Set `TEST_INTEGRATION=1`
-* Run `cargo test`
+**CAUTION:** existing data in the database is dropped when tests are run, so you should use a DIFFERENT
+database name for your test database than your `INFLUXDB_IOX_CATALOG_DSN` database.
 
-**CAUTION:** existing data in the database is dropped when tests are run
+* Set `TEST_INFLUXDB_IOX_CATALOG_DSN=<testdsn>` env as above with the `INFLUXDB_IOX_CATALOG_DSN` env var. The
+  integration tests *will* pick up this value if set in your `.env` file.
+* Create the test database once by running:
+
+```
+DATABASE_URL=<testdsn> sqlx database create
+```
+
+* Set up the test database as above but specify the test database URL as follows (`catalog setup`
+  will NOT pick up the `TEST_` version of the environment variable from your `.env` file so it needs to be
+  specified explicitly WITHOUT the `TEST_` prefix):
+
+```
+INFLUXDB_IOX_CATALOG_DSN=<testdsn> cargo run -q -- catalog setup
+INFLUXDB_IOX_CATALOG_DSN=<testdsn> cargo run -q -- catalog topic update iox-shared
+```
+
+* Set `TEST_INTEGRATION=1`
+* Run `cargo test -p iox_catalog`
 
 ## Schema namespace
 
@@ -49,7 +74,7 @@ All iox catalog tables are created in a `iox_catalog` schema. Remember to set th
 There are several ways to set the default search path, depending if you want to do it for your session, for the database or for the user.
 
 Setting a default search path for the database or user may interfere with tests (e.g. it may make some test pass when they should fail).
-The safest option is set the search path on a per session basis. As always, there are a few ways to do that: 
+The safest option is set the search path on a per session basis. As always, there are a few ways to do that:
 
 1. you can type `set search_path to public,iox_catalog;` inside psql.
 2. you can add (1) to your `~/.psqlrc`

iox_catalog/src/postgres.rs

@@ -1369,12 +1369,13 @@ mod tests {
     use rand::Rng;
     use tempfile::NamedTempFile;
 
-    // Helper macro to skip tests if TEST_INTEGRATION and the AWS environment variables are not set.
+    // Helper macro to skip tests if TEST_INTEGRATION and TEST_INFLUXDB_IOX_CATALOG_DSN environment variables
+    // are not set.
     macro_rules! maybe_skip_integration {
         ($panic_msg:expr) => {{
             dotenv::dotenv().ok();
 
-            let required_vars = ["DATABASE_URL"];
+            let required_vars = ["TEST_INFLUXDB_IOX_CATALOG_DSN"];
             let unset_vars: Vec<_> = required_vars
                 .iter()
                 .filter_map(|&name| match env::var(name) {
@@ -1442,7 +1443,7 @@ mod tests {
         };
 
         let metrics = Arc::new(metric::Registry::default());
-        let dsn = std::env::var("DATABASE_URL").unwrap();
+        let dsn = std::env::var("TEST_INFLUXDB_IOX_CATALOG_DSN").unwrap();
         let pg = PostgresCatalog::connect("test", &schema_name, &dsn, 3, metrics)
             .await
             .expect("failed to connect catalog");
@@ -1780,7 +1781,7 @@ mod tests {
         const POLLING_INTERVAL: Duration = Duration::from_millis(10);
 
         // fetch dsn from envvar
-        let test_dsn = std::env::var("DATABASE_URL").unwrap();
+        let test_dsn = std::env::var("TEST_INFLUXDB_IOX_CATALOG_DSN").unwrap();
         eprintln!("TEST_DSN={}", test_dsn);
 
         // create a temp file to store the initial dsn

sqlx-hotswap-pool/src/lib.rs

@@ -144,12 +144,13 @@ mod tests {
     use rand::{distributions::Alphanumeric, Rng};
     use sqlx::{postgres::PgPoolOptions, Postgres};
 
-    // Helper macro to skip tests if TEST_INTEGRATION and the AWS environment variables are not set.
+    // Helper macro to skip tests if TEST_INTEGRATION and TEST_INFLUXDB_IOX_CATALOG_DSN environment variables
+    // are not set.
     macro_rules! maybe_skip_integration {
         () => {{
             dotenv::dotenv().ok();
 
-            let required_vars = ["DATABASE_URL"];
+            let required_vars = ["TEST_INFLUXDB_IOX_CATALOG_DSN"];
             let unset_vars: Vec<_> = required_vars
                 .iter()
                 .filter_map(|&name| match env::var(name) {
@@ -195,7 +196,7 @@ mod tests {
                 .map(char::from)
                 .collect::<String>()
         };
-        let dsn = std::env::var("DATABASE_URL").unwrap();
+        let dsn = std::env::var("TEST_INFLUXDB_IOX_CATALOG_DSN").unwrap();
         let captured_schema_name = schema_name.clone();
         PgPoolOptions::new()
             .min_connections(1)