diff --git a/Cargo.toml b/Cargo.toml
index fe4af87b13..f26e75b722 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -73,6 +73,7 @@ exclude = [
     "massif.out.*",
     "perf/",
     "scripts/",
+    "test_bench/",
     "test_fixtures/",
     "tools/",
 ]
diff --git a/test_bench/README.md b/test_bench/README.md
new file mode 100644
index 0000000000..d0e372ac2a
--- /dev/null
+++ b/test_bench/README.md
@@ -0,0 +1,43 @@
+# Test Bench
+
+The test bench makes it easier to stage deployment scenarios to:
+
+- experiment and test with novel multi-node architectures
+- simulate workarounds beyond what is possible on developer machines
+- test complex interplay between components
+
+## Architecture
+The test bench is meant to run in [Kubernetes]. To create different test setups or to customize a setup to a specific
+cluster we use [Kustomize]. The [Kustomize] configs can be found in `k8s/`.
+
+To glue IOx to [Kubernetes] we need some helper code which can be found under `k8s/base/glue-entrypoint.bash`.
+
+## Getting Started
+We'll "kustomize" the test bench. You can use `k8s/overlays/demo` as a starting point. You may wanna
+[adjust images](https://kubectl.docs.kubernetes.io/references/kustomize/kustomization/images/) or change the router and
+database configs. Then you can deploy the whole thing:
+
+```console
+$ kubectl kustomize ./k8s/overlays/demo| kubectl apply -f -
+```
+
+Now IOx should be ready for experiments. You might just feed it with some data:
+
+```console
+$ # in another terminal:
+$ kubectl port-forward service/iox-router-service 8080:8080 8082:8082
+
+$ # in your main terminal:
+$ cd ../iox_data_generator/
+$ cargo run --release -- \
+    --spec schemas/cap-write.toml \
+    --continue \
+    --host 127.0.0.1:8080 \
+    --token arbitrary \
+    --org myorg \
+    --bucket mybucket
+```
+
+
+[Kustomize]: https://kustomize.io/
+[Kubernetes]: https://kubernetes.io/
diff --git a/test_bench/k8s/base/debug-sidecar-patch.yml b/test_bench/k8s/base/debug-sidecar-patch.yml
new file mode 100644
index 0000000000..43bfe86159
--- /dev/null
+++ b/test_bench/k8s/base/debug-sidecar-patch.yml
@@ -0,0 +1,29 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: not-important
+spec:
+  template:
+    spec:
+      containers:
+        - name: debug
+          image: nixery.dev/shell/findutils/gnugrep/gnutar/hexdump/less/mount/procps/curl/grpcurl/binutils/gdb/strace/rustc/linuxpackages.perf/perf-tools/gzip/zstd
+          command:
+            - /bin/bash
+            - -c
+            - --
+          args:
+            - while true; do sleep 30; done;
+          resources:
+            requests:
+              cpu: 1
+              memory: 1G
+            limits:
+              cpu: 1
+              memory: 2G
+          securityContext:
+            capabilities:
+              add:
+                - SYS_PTRACE
+      shareProcessNamespace: true
diff --git a/test_bench/k8s/base/glue-entrypoint.bash b/test_bench/k8s/base/glue-entrypoint.bash
new file mode 100755
index 0000000000..afcaf12365
--- /dev/null
+++ b/test_bench/k8s/base/glue-entrypoint.bash
@@ -0,0 +1,99 @@
+#!/bin/bash
+
+set -eu -o pipefail
+shopt -s expand_aliases
+
+# extract ordinal index from server ID
+[[ $HOSTNAME =~ -([0-9]+)$ ]] || (echo "invalid hostname" && exit 1)
+ordinal=${BASH_REMATCH[1]}
+
+# calculate server ID
+offset="${INFLUXDB_IOX_ID_OFFSET:-1}"
+server_id=$((ordinal + offset))
+echo "ServerID: $server_id"
+
+# set server ID
+alias textgrpc="grpcurl -allow-unknown-fields -plaintext"
+while true; do
+    if textgrpc -d "{\"id\": $server_id}" "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.deployment.v1.DeploymentService.UpdateServerId; then
+        echo "server ID set"
+        break
+    else
+        echo "cannot set server ID yet, waiting..."
+        sleep 1
+    fi
+done
+
+# wait for rule/config updates
+FINGERPRINT_DONE=/fingerprint.done
+FINGERPRINT_STAGING=/fingerprint.staging
+F_CURRENT=/current
+while true; do
+    # create new fingerprint
+    rm -rf "$FINGERPRINT_STAGING"
+    mkdir "$FINGERPRINT_STAGING"
+
+    for cfg_file in /iox_config/*; do
+        # create backup of file so it doesn't change while we're working with it
+        cp "$cfg_file" "$F_CURRENT"
+
+        # determine type
+        if [[ $cfg_file == *"router"* ]]; then
+            type="router"
+        else
+            type="database"
+        fi
+
+        # compare hash
+        hash="$(sha256sum "$F_CURRENT" | sed -E 's/([a-f0-9]+).*/\1/g')"
+        in_sync=0
+        if [ -f "$FINGERPRINT_DONE/$hash.$type" ]; then
+            echo "$cfg_file: in-sync"
+            in_sync=1
+        else
+            echo "$cfg_file: out of sync"
+
+            # select create/update routing depending on the config type
+            if [[ $type == "database" ]]; then
+                echo "Create database..."
+
+                if textgrpc -d @ < "$F_CURRENT" "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.management.v1.ManagementService.CreateDatabase; then
+                    echo "databse created"
+                    in_sync=1
+                else
+                    echo "cannot create database, try updating it..."
+
+                    if textgrpc -d @ < "$F_CURRENT" "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.management.v1.ManagementService.UpdateDatabase; then
+                        echo "database updated"
+                        in_sync=1
+                    else
+                        echo "cannot update database"
+                    fi
+                fi
+            else
+                echo "Update router..."
+
+                if textgrpc -d @ < "$F_CURRENT" "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.router.v1.RouterService.UpdateRouter; then
+                    echo "router updated"
+                    in_sync=1
+                else
+                    echo "cannot update router"
+                fi
+            fi
+        fi
+
+        # store state
+        if [[ $in_sync == 1 ]]; then
+            cp "$F_CURRENT" "$FINGERPRINT_STAGING/$hash.$type"
+        fi
+    done
+
+    # TODO: delete resources that are no longer present, should be done by comparing the list of resources on the server VS the ones in config files by name
+
+    # store fingerprints for next round
+    rm -rf "$FINGERPRINT_DONE"
+    mkdir "$FINGERPRINT_DONE"
+    cp -r "$FINGERPRINT_STAGING"/. "$FINGERPRINT_DONE"/
+
+    sleep 10
+done
diff --git a/test_bench/k8s/base/kustomization.yml b/test_bench/k8s/base/kustomization.yml
new file mode 100644
index 0000000000..707bb47990
--- /dev/null
+++ b/test_bench/k8s/base/kustomization.yml
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - minio.yml
+  - redpanda.yml
+  - router.yml
+  - query.yml
+
+configMapGenerator:
+  - name: glue-entrypoint
+    files:
+      - glue-entrypoint.bash
+
+patches:
+  - path: debug-sidecar-patch.yml
+    target:
+      kind: StatefulSet
+      name: iox-.*
diff --git a/test_bench/k8s/base/minio.yml b/test_bench/k8s/base/minio.yml
new file mode 100644
index 0000000000..65eb7a4796
--- /dev/null
+++ b/test_bench/k8s/base/minio.yml
@@ -0,0 +1,69 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio-deployment
+  labels:
+    app: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      containers:
+        - name: minio
+          image: minio/minio:RELEASE.2021-08-05T22-01-19Z
+          args:
+            - server
+            - --address=0.0.0.0:8080
+            - --console-address=0.0.0.0:8081
+            - /data
+          env:
+            - name: MINIO_ROOT_USER
+              value: minio
+            - name: MINIO_ROOT_PASSWORD
+              value: miniominio
+            - name: MINIO_PROMETHEUS_AUTH_TYPE
+              value: public
+            - name: MINIO_HTTP_TRACE
+              value: /dev/stdout
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      initContainers:
+        - name: create-bucket
+          image: minio/minio:RELEASE.2021-08-05T22-01-19Z
+          command:
+            - /bin/sh
+          args:
+            - -c
+            - mkdir -p /data/iox
+          volumeMounts:
+            - name: data
+              mountPath: /data
+      volumes:
+        - name: data
+          emptyDir: {}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio-service
+spec:
+  selector:
+    app: minio
+  ports:
+    - name: s3
+      protocol: TCP
+      port: 8080
+      targetPort: 8080
+    - name: console
+      protocol: TCP
+      port: 8081
+      targetPort: 8081
diff --git a/test_bench/k8s/base/query.yml b/test_bench/k8s/base/query.yml
new file mode 100644
index 0000000000..74f7bde31d
--- /dev/null
+++ b/test_bench/k8s/base/query.yml
@@ -0,0 +1,100 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iox-query-statefulset
+spec:
+  selector:
+    matchLabels:
+      app: iox-query
+  serviceName: "iox-query"
+  replicas: 2
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: iox
+      labels:
+        app: iox-query
+    spec:
+      containers:
+        - name: iox
+          image: quay.io/influxdb/iox:main
+          imagePullPolicy: Always
+          env:
+            - name: INFLUXDB_IOX_SERVER_MODE
+              value: database
+            - name: INFLUXDB_IOX_BIND_ADDR
+              value: 0.0.0.0:8080
+            - name: INFLUXDB_IOX_GRPC_BIND_ADDR
+              value: 0.0.0.0:8082
+            - name: LOG_FILTER
+              value: info
+            - name: RUST_BACKTRACE
+              value: "1"
+            - name: RUST_LOG
+              value: info
+            - name: INFLUXDB_IOX_OBJECT_STORE
+              value: s3
+            - name: INFLUXDB_IOX_BUCKET
+              value: iox
+            - name: AWS_ACCESS_KEY_ID
+              value: minio
+            - name: AWS_SECRET_ACCESS_KEY
+              value: miniominio
+            - name: AWS_ENDPOINT
+              value: http://minio-service:8080
+          resources:
+            requests:
+              cpu: 1
+              memory: 1G
+            limits:
+              cpu: 1
+              memory: 2G
+        - name: glue
+          image: nixery.dev/shell/gnused/grpcurl
+          imagePullPolicy: Always
+          command:
+            - /bin/bash
+            - /entrypoint/glue-entrypoint.bash
+          env:
+            - name: INFLUXDB_IOX_ID_OFFSET
+              value: "1000"
+            - name: INFLUXDB_IOX_GRPC_BIND_ADDR
+              value: 0.0.0.0:8082
+          resources:
+            requests:
+              cpu: 1
+              memory: 1G
+            limits:
+              cpu: 1
+              memory: 2G
+          volumeMounts:
+            - name: config
+              mountPath: /iox_config
+            - name: glue-entrypoint
+              mountPath: /entrypoint
+      volumes:
+        - name: config
+          configMap:
+            name: iox-query-configs
+        - name: glue-entrypoint
+          configMap:
+            name: glue-entrypoint
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iox-query-service
+spec:
+  selector:
+    app: iox-query
+  ports:
+    - name: http
+      protocol: TCP
+      port: 8080
+      targetPort: 8080
+    - name: grpc
+      protocol: TCP
+      port: 8082
+      targetPort: 8082
diff --git a/test_bench/k8s/base/redpanda.yml b/test_bench/k8s/base/redpanda.yml
new file mode 100644
index 0000000000..63e80cc357
--- /dev/null
+++ b/test_bench/k8s/base/redpanda.yml
@@ -0,0 +1,45 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redpanda-deployment
+  labels:
+    app: redpanda
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redpanda
+  template:
+    metadata:
+      labels:
+        app: redpanda
+    spec:
+      containers:
+        - name: redpanda
+          image: vectorized/redpanda:v21.7.6
+          args:
+            - redpanda
+            - start
+            - --overprovisioned
+            - --smp=1
+            - --memory=128M
+            - --reserve-memory=0M
+            - --node-id=0
+            - --check=false
+            - --kafka-addr=CLIENT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093
+            - --advertise-kafka-addr=CLIENT://redpanda-service:9092,EXTERNAL://redpanda-service:9093
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: redpanda-service
+spec:
+  selector:
+    app: redpanda
+  ports:
+    - name: kafka
+      protocol: TCP
+      port: 9093
+      targetPort: 9093
diff --git a/test_bench/k8s/base/router.yml b/test_bench/k8s/base/router.yml
new file mode 100644
index 0000000000..aa08a975c4
--- /dev/null
+++ b/test_bench/k8s/base/router.yml
@@ -0,0 +1,90 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: iox-router-statefulset
+spec:
+  selector:
+    matchLabels:
+      app: iox-router
+  serviceName: "iox-router"
+  replicas: 2
+  template:
+    metadata:
+      annotations:
+        kubectl.kubernetes.io/default-container: iox
+      labels:
+        app: iox-router
+    spec:
+      containers:
+        - name: iox
+          image: quay.io/influxdb/iox:main
+          imagePullPolicy: Always
+          env:
+            - name: INFLUXDB_IOX_SERVER_MODE
+              value: router
+            - name: INFLUXDB_IOX_BIND_ADDR
+              value: 0.0.0.0:8080
+            - name: INFLUXDB_IOX_GRPC_BIND_ADDR
+              value: 0.0.0.0:8082
+            - name: LOG_FILTER
+              value: info
+            - name: RUST_BACKTRACE
+              value: "1"
+            - name: RUST_LOG
+              value: info
+          resources:
+            requests:
+              cpu: 1
+              memory: 1G
+            limits:
+              cpu: 1
+              memory: 2G
+        - name: glue
+          image: nixery.dev/shell/gnused/grpcurl
+          imagePullPolicy: Always
+          command:
+            - /bin/bash
+            - /entrypoint/glue-entrypoint.bash
+          env:
+            - name: INFLUXDB_IOX_ID_OFFSET
+              value: "1000"
+            - name: INFLUXDB_IOX_GRPC_BIND_ADDR
+              value: 0.0.0.0:8082
+          resources:
+            requests:
+              cpu: 1
+              memory: 1G
+            limits:
+              cpu: 1
+              memory: 2G
+          volumeMounts:
+            - name: config
+              mountPath: /iox_config
+            - name: glue-entrypoint
+              mountPath: /entrypoint
+      volumes:
+        - name: config
+          configMap:
+            name: iox-router-configs
+        - name: glue-entrypoint
+          configMap:
+            name: glue-entrypoint
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: iox-router-service
+spec:
+  selector:
+    app: iox-router
+  ports:
+    - name: http
+      protocol: TCP
+      port: 8080
+      targetPort: 8080
+    - name: grpc
+      protocol: TCP
+      port: 8082
+      targetPort: 8082
diff --git a/test_bench/k8s/overlays/demo/database-1.json b/test_bench/k8s/overlays/demo/database-1.json
new file mode 100644
index 0000000000..ba8ab1c518
--- /dev/null
+++ b/test_bench/k8s/overlays/demo/database-1.json
@@ -0,0 +1,34 @@
+{
+    "rules": {
+        "name": "myorg_mybucket",
+        "partition_template": {
+            "parts": [
+                {"time": "%Y-%m-%d %H:00:00"}
+            ]
+        },
+        "lifecycle_rules": {
+            "buffer_size_soft": 1073741824,
+            "buffer_size_hard": 2147483648,
+            "worker_backoff_millis": 100,
+            "max_active_compactions": 1,
+            "persist": true,
+            "persist_row_threshold": 10000000,
+            "catalog_transactions_until_checkpoint": 100,
+            "late_arrive_window_seconds": 300,
+            "persist_age_threshold_seconds": 1800,
+            "mub_row_threshold": 100000
+        },
+        "routing_config": {"sink": {"kafka": {}}},
+        "worker_cleanup_avg_sleep": "500s",
+        "write_buffer_connection": {
+            "direction": "DIRECTION_READ",
+            "type": "kafka",
+            "connection": "redpanda-service:9093",
+            "connection_config": {},
+            "creation_config": {
+                "n_sequencers": 1,
+                "options": {}
+            }
+        }
+    }
+}
diff --git a/test_bench/k8s/overlays/demo/kustomization.yaml b/test_bench/k8s/overlays/demo/kustomization.yaml
new file mode 100644
index 0000000000..74e0b512f7
--- /dev/null
+++ b/test_bench/k8s/overlays/demo/kustomization.yaml
@@ -0,0 +1,20 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../base
+
+configMapGenerator:
+  - name: iox-query-configs
+    files:
+      - database-1.json
+    options:
+      # we don't wanna recreate containers when configs change
+      disableNameSuffixHash: true
+  - name: iox-router-configs
+    files:
+      - router-1.json
+    options:
+      # we don't wanna recreate containers when configs change
+      disableNameSuffixHash: true
diff --git a/test_bench/k8s/overlays/demo/router-1.json b/test_bench/k8s/overlays/demo/router-1.json
new file mode 100644
index 0000000000..726388b919
--- /dev/null
+++ b/test_bench/k8s/overlays/demo/router-1.json
@@ -0,0 +1,36 @@
+{
+  "router": {
+      "name": "myorg_mybucket",
+      "write_sharder": {
+          "specific_targets": [
+              {
+                  "matcher": {
+                      "table_name_regex": ".*"
+                  },
+                  "shard": 1
+              }
+          ],
+          "hash_ring": null
+      },
+      "write_sinks": {
+          "1": {
+              "sinks": [
+                  {
+                      "write_buffer": {
+                          "direction": "DIRECTION_WRITE",
+                          "type": "kafka",
+                          "connection": "redpanda-service:9093",
+                          "connection_config": {},
+                          "creation_config": {
+                              "n_sequencers": 1,
+                              "options": {}
+                          }
+                      },
+                      "ignore_errors": false
+                  }
+              ]
+          }
+      },
+      "query_sinks": null
+  }
+}