feat: test bench
Marco Neumann
parent
524dfa8ce2
commit
f2f090bde7
|
|
@ -73,6 +73,7 @@ exclude = [
|
|||
"massif.out.*",
|
||||
"perf/",
|
||||
"scripts/",
|
||||
"test_bench/",
|
||||
"test_fixtures/",
|
||||
"tools/",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,57 @@
|
|||
# Test Bench
|
||||
|
||||
The test bench makes it easier to stage deployment scenarios to:
|
||||
|
||||
- experiment and test with novel multi-node architectures
|
||||
- simulate workarounds beyond what is possible on developer machines
|
||||
- test complex interplay between components
|
||||
|
||||
## Architecture
|
||||
The test bench is meant to run in [Kubernetes]. To create different test setups or to customize a setup to a specific
|
||||
cluster we use [Kustomize]. The [Kustomize] configs can be found in `k8s/`.
|
||||
|
||||
To glue IOx to [Kubernetes] we need some helper code which can be found under `glue/`.
|
||||
|
||||
## Getting Started
|
||||
First you need to build an image with the glue code:
|
||||
|
||||
```console
|
||||
$ cd glue
|
||||
$ docker build -t iox-glue .
|
||||
$ cd ..
|
||||
```
|
||||
|
||||
Then push the image to some container registry which is accessible by your [Kubernetes] cluster:
|
||||
|
||||
```console
|
||||
$ docker push some.registry.io/foo/iox-glue
|
||||
```
|
||||
|
||||
Next we'll "kustomize" the test bench. You can use `k8s/overlays/demo` as a starting point. There you should at least
|
||||
have a look at `images-patch.yml` to set the image with the glue code that you've just built. Then you can deploy the
|
||||
whole thing:
|
||||
|
||||
```console
|
||||
$ kubectl kustomize ./k8s/overlays/demo| kubectl apply -f -
|
||||
```
|
||||
|
||||
Now IOx should be ready for experiments. You might just feed it with some data:
|
||||
|
||||
```console
|
||||
$ # in another terminal:
|
||||
$ kubectl port-forward service/iox-router-service 8080:8080 8082:8082
|
||||
|
||||
$ # in your main terminal:
|
||||
$ cd ../iox_data_generator/
|
||||
$ cargo run --release -- \
|
||||
--spec schemas/cap-write.toml \
|
||||
--continue \
|
||||
--host 127.0.0.1:8080 \
|
||||
--token arbitrary \
|
||||
--org myorg \
|
||||
--bucket mybucket
|
||||
```
|
||||
|
||||
|
||||
[Kustomize]: https://kustomize.io/
|
||||
[Kubernetes]: https://kubernetes.io/
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
FROM alpine:latest
|
||||
|
||||
ENV CURL_FLAGS="--proto =https --tlsv1.2 -sSf" \
|
||||
GRPCURL_VERSION="1.8.5"
|
||||
|
||||
RUN apk add --no-cache bash curl
|
||||
|
||||
RUN curl ${CURL_FLAGS} -Lo grpcurl.tar.gz https://github.com/fullstorydev/grpcurl/releases/download/v${GRPCURL_VERSION}/grpcurl_${GRPCURL_VERSION}_linux_x86_64.tar.gz \
|
||||
&& tar xvf grpcurl.tar.gz \
|
||||
&& mv grpcurl /bin
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/entrypoint.sh"]
|
||||
|
|
@ -0,0 +1,119 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eu -o pipefail
|
||||
|
||||
# extract ordinal index from server ID
|
||||
[[ $HOSTNAME =~ -([0-9]+)$ ]] || (echo "invalid hostname" && exit 1)
|
||||
ordinal=${BASH_REMATCH[1]}
|
||||
|
||||
# calculate server ID
|
||||
offset="${INFLUXDB_IOX_ID_OFFSET:-0}"
|
||||
server_id=$((ordinal + offset))
|
||||
echo "ServerID: $server_id"
|
||||
|
||||
# set server ID
|
||||
while true; do
|
||||
set +e
|
||||
grpcurl -d "{\"id\": $server_id}" -allow-unknown-fields -plaintext "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.deployment.v1.DeploymentService.UpdateServerId
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status == 0 ]]; then
|
||||
echo "server ID set"
|
||||
break
|
||||
else
|
||||
echo "cannot set server ID yet, waiting..."
|
||||
sleep 1
|
||||
fi
|
||||
done
|
||||
|
||||
# wait for rule/config updates
|
||||
FINGERPRINT_DONE=/fingerprint.done
|
||||
FINGERPRINT_STAGING=/fingerprint.staging
|
||||
F_CURRENT=/current
|
||||
while true; do
|
||||
# create new fingerprint
|
||||
rm -rf "$FINGERPRINT_STAGING"
|
||||
mkdir "$FINGERPRINT_STAGING"
|
||||
|
||||
for cfg_file in /iox_config/*; do
|
||||
# create backup of file so it doesn't change while we're working with it
|
||||
cp "$cfg_file" "$F_CURRENT"
|
||||
|
||||
# determine type
|
||||
if [[ $cfg_file == *"router"* ]]; then
|
||||
type="router"
|
||||
else
|
||||
type="database"
|
||||
fi
|
||||
|
||||
# compare hash
|
||||
hash="$(sha256sum "$F_CURRENT" | sed -E 's/([a-f0-9]+).*/\1/g')"
|
||||
in_sync=0
|
||||
if [ -f "$FINGERPRINT_DONE/$hash.$type" ]; then
|
||||
echo "$cfg_file: in-sync"
|
||||
in_sync=1
|
||||
else
|
||||
echo "$cfg_file: out of sync"
|
||||
|
||||
# select create/update routing depending on the config type
|
||||
if [[ $type == "database" ]]; then
|
||||
echo "Create database..."
|
||||
|
||||
set +e
|
||||
grpcurl -d @ < "$F_CURRENT" -allow-unknown-fields -plaintext "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.management.v1.ManagementService.CreateDatabase
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status == 0 ]]; then
|
||||
echo "databse created"
|
||||
in_sync=1
|
||||
else
|
||||
echo "cannot create database, try updating it..."
|
||||
|
||||
set +e
|
||||
grpcurl -d @ < "$F_CURRENT" -allow-unknown-fields -plaintext "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.management.v1.ManagementService.UpdateDatabase
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status == 0 ]]; then
|
||||
echo "database updated"
|
||||
in_sync=1
|
||||
else
|
||||
echo "cannot update database"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo "Update router..."
|
||||
|
||||
set +e
|
||||
grpcurl -d @ < "$F_CURRENT" -allow-unknown-fields -plaintext "$INFLUXDB_IOX_GRPC_BIND_ADDR" influxdata.iox.router.v1.RouterService.UpdateRouter
|
||||
status=$?
|
||||
set -e
|
||||
|
||||
if [[ $status == 0 ]]; then
|
||||
echo "router updated"
|
||||
in_sync=1
|
||||
else
|
||||
echo "cannot update router"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# store state
|
||||
if [[ $in_sync == 1 ]]; then
|
||||
cp "$F_CURRENT" "$FINGERPRINT_STAGING/$hash.$type"
|
||||
fi
|
||||
done
|
||||
|
||||
# TODO: delete resources that are no longer present, should be done by comparing the list of resources on the server VS the ones in config files by name
|
||||
|
||||
# store fingerprints for next round
|
||||
rm -rf "$FINGERPRINT_DONE"
|
||||
mkdir "$FINGERPRINT_DONE"
|
||||
if [ -n "$(ls -A "$FINGERPRINT_STAGING")" ]; then
|
||||
cp "$FINGERPRINT_STAGING"/* "$FINGERPRINT_DONE"/
|
||||
fi
|
||||
|
||||
sleep 10
|
||||
done
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: not-important
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: debug
|
||||
image: nixery.dev/shell/findutils/gnugrep/gnutar/hexdump/less/mount/procps/curl/grpcurl/binutils/gdb/strace/rustc/linuxpackages.perf/perf-tools/gzip/zstd
|
||||
command:
|
||||
- /bin/bash
|
||||
- -c
|
||||
- --
|
||||
args:
|
||||
- while true; do sleep 30; done;
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 1G
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2G
|
||||
securityContext:
|
||||
capabilities:
|
||||
add:
|
||||
- SYS_PTRACE
|
||||
shareProcessNamespace: true
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- minio.yml
|
||||
- redpanda.yml
|
||||
- router.yml
|
||||
- query.yml
|
||||
|
||||
patches:
|
||||
- path: debug-sidecar-patch.yml
|
||||
target:
|
||||
kind: StatefulSet
|
||||
name: iox-.*
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: minio-deployment
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: minio
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: minio
|
||||
spec:
|
||||
containers:
|
||||
- name: minio
|
||||
image: minio/minio:RELEASE.2021-08-05T22-01-19Z
|
||||
args:
|
||||
- server
|
||||
- --address=0.0.0.0:8080
|
||||
- --console-address=0.0.0.0:8081
|
||||
- /data
|
||||
env:
|
||||
- name: MINIO_ROOT_USER
|
||||
value: minio
|
||||
- name: MINIO_ROOT_PASSWORD
|
||||
value: miniominio
|
||||
- name: MINIO_PROMETHEUS_AUTH_TYPE
|
||||
value: public
|
||||
- name: MINIO_HTTP_TRACE
|
||||
value: /dev/stdout
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
initContainers:
|
||||
- name: create-bucket
|
||||
image: minio/minio:RELEASE.2021-08-05T22-01-19Z
|
||||
command:
|
||||
- /bin/sh
|
||||
args:
|
||||
- -c
|
||||
- mkdir -p /data/iox
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /data
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: minio-service
|
||||
spec:
|
||||
selector:
|
||||
app: minio
|
||||
ports:
|
||||
- name: s3
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
- name: console
|
||||
protocol: TCP
|
||||
port: 8081
|
||||
targetPort: 8081
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: iox-query-statefulset
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: iox-query
|
||||
serviceName: "iox-query"
|
||||
replicas: 2
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kubectl.kubernetes.io/default-container: iox
|
||||
labels:
|
||||
app: iox-query
|
||||
spec:
|
||||
containers:
|
||||
- name: iox
|
||||
env:
|
||||
- name: INFLUXDB_IOX_SERVER_MODE
|
||||
value: database
|
||||
- name: INFLUXDB_IOX_BIND_ADDR
|
||||
value: 0.0.0.0:8080
|
||||
- name: INFLUXDB_IOX_GRPC_BIND_ADDR
|
||||
value: 0.0.0.0:8082
|
||||
- name: LOG_FILTER
|
||||
value: info
|
||||
- name: RUST_BACKTRACE
|
||||
value: "1"
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
- name: INFLUXDB_IOX_OBJECT_STORE
|
||||
value: s3
|
||||
- name: INFLUXDB_IOX_BUCKET
|
||||
value: iox
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
value: minio
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
value: miniominio
|
||||
- name: AWS_ENDPOINT
|
||||
value: http://minio-service:8080
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 1G
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2G
|
||||
- name: glue
|
||||
env:
|
||||
- name: INFLUXDB_IOX_ID_OFFSET
|
||||
value: "1"
|
||||
- name: INFLUXDB_IOX_GRPC_BIND_ADDR
|
||||
value: 0.0.0.0:8082
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 1G
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2G
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /iox_config
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: iox-query-configs
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: iox-query-service
|
||||
spec:
|
||||
selector:
|
||||
app: iox-query
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
- name: grpc
|
||||
protocol: TCP
|
||||
port: 8082
|
||||
targetPort: 8082
|
||||
|
|
@ -0,0 +1,45 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: redpanda-deployment
|
||||
labels:
|
||||
app: redpanda
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: redpanda
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: redpanda
|
||||
spec:
|
||||
containers:
|
||||
- name: redpanda
|
||||
image: vectorized/redpanda:v21.7.6
|
||||
args:
|
||||
- redpanda
|
||||
- start
|
||||
- --overprovisioned
|
||||
- --smp=1
|
||||
- --memory=128M
|
||||
- --reserve-memory=0M
|
||||
- --node-id=0
|
||||
- --check=false
|
||||
- --kafka-addr=CLIENT://0.0.0.0:9092,EXTERNAL://0.0.0.0:9093
|
||||
- --advertise-kafka-addr=CLIENT://redpanda-service:9092,EXTERNAL://redpanda-service:9093
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redpanda-service
|
||||
spec:
|
||||
selector:
|
||||
app: redpanda
|
||||
ports:
|
||||
- name: kafka
|
||||
protocol: TCP
|
||||
port: 9093
|
||||
targetPort: 9093
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: iox-router-statefulset
|
||||
spec:
|
||||
selector:
|
||||
matchLabels:
|
||||
app: iox-router
|
||||
serviceName: "iox-router"
|
||||
replicas: 2
|
||||
template:
|
||||
metadata:
|
||||
annotations:
|
||||
kubectl.kubernetes.io/default-container: iox
|
||||
labels:
|
||||
app: iox-router
|
||||
spec:
|
||||
containers:
|
||||
- name: iox
|
||||
env:
|
||||
- name: INFLUXDB_IOX_SERVER_MODE
|
||||
value: router
|
||||
- name: INFLUXDB_IOX_BIND_ADDR
|
||||
value: 0.0.0.0:8080
|
||||
- name: INFLUXDB_IOX_GRPC_BIND_ADDR
|
||||
value: 0.0.0.0:8082
|
||||
- name: LOG_FILTER
|
||||
value: info
|
||||
- name: RUST_BACKTRACE
|
||||
value: "1"
|
||||
- name: RUST_LOG
|
||||
value: info
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 1G
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2G
|
||||
- name: glue
|
||||
env:
|
||||
- name: INFLUXDB_IOX_ID_OFFSET
|
||||
value: "1000"
|
||||
- name: INFLUXDB_IOX_GRPC_BIND_ADDR
|
||||
value: 0.0.0.0:8082
|
||||
resources:
|
||||
requests:
|
||||
cpu: 1
|
||||
memory: 1G
|
||||
limits:
|
||||
cpu: 1
|
||||
memory: 2G
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /iox_config
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: iox-router-configs
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: iox-router-service
|
||||
spec:
|
||||
selector:
|
||||
app: iox-router
|
||||
ports:
|
||||
- name: http
|
||||
protocol: TCP
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
- name: grpc
|
||||
protocol: TCP
|
||||
port: 8082
|
||||
targetPort: 8082
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: iox-router-configs
|
||||
data:
|
||||
router-1: |
|
||||
{
|
||||
"router": {
|
||||
"name": "myorg_mybucket",
|
||||
"write_sharder": {
|
||||
"specific_targets": [
|
||||
{
|
||||
"matcher": {
|
||||
"table_name_regex": ".*"
|
||||
},
|
||||
"shard": 1
|
||||
}
|
||||
],
|
||||
"hash_ring": null
|
||||
},
|
||||
"write_sinks": {
|
||||
"1": {
|
||||
"sinks": [
|
||||
{
|
||||
"write_buffer": {
|
||||
"direction": "DIRECTION_WRITE",
|
||||
"type": "kafka",
|
||||
"connection": "redpanda-service:9093",
|
||||
"connection_config": {},
|
||||
"creation_config": {
|
||||
"n_sequencers": 1,
|
||||
"options": {}
|
||||
}
|
||||
},
|
||||
"ignore_errors": false
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"query_sinks": null
|
||||
}
|
||||
}
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: iox-query-configs
|
||||
data:
|
||||
database-1: |
|
||||
{
|
||||
"rules": {
|
||||
"name": "myorg_mybucket",
|
||||
"partition_template": {
|
||||
"parts": [
|
||||
{"time": "%Y-%m-%d %H:00:00"}
|
||||
]
|
||||
},
|
||||
"lifecycle_rules": {
|
||||
"buffer_size_soft": 1073741824,
|
||||
"buffer_size_hard": 2147483648,
|
||||
"worker_backoff_millis": 100,
|
||||
"max_active_compactions": 1,
|
||||
"persist": true,
|
||||
"persist_row_threshold": 10000000,
|
||||
"catalog_transactions_until_checkpoint": 100,
|
||||
"late_arrive_window_seconds": 300,
|
||||
"persist_age_threshold_seconds": 1800,
|
||||
"mub_row_threshold": 100000
|
||||
},
|
||||
"routing_config": {"sink": {"kafka": {}}},
|
||||
"worker_cleanup_avg_sleep": "500s",
|
||||
"write_buffer_connection": {
|
||||
"direction": "DIRECTION_READ",
|
||||
"type": "kafka",
|
||||
"connection": "redpanda-service:9093",
|
||||
"connection_config": {},
|
||||
"creation_config": {
|
||||
"n_sequencers": 1,
|
||||
"options": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: not-important
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: iox
|
||||
image: quay.io/influxdb/iox:main
|
||||
imagePullPolicy: Always
|
||||
- name: glue
|
||||
image: europe-west3-docker.pkg.dev/influxdata-fusion/mneumann-repository-docker-1/iox-glue
|
||||
imagePullPolicy: Always
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
---
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
resources:
|
||||
- ../../base
|
||||
- configs.yml
|
||||
|
||||
patches:
|
||||
- path: images-patch.yml
|
||||
target:
|
||||
kind: StatefulSet
|
||||
name: iox-.*
|
||||
Loading…
Reference in New Issue