feat(docker): Rework docker builds to push to quay and be multi-arch. (#25800)

.circleci/config.yml

@@ -85,6 +85,11 @@ commands:
           command: |
             gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file <(echo "${GCLOUD_SERVICE_ACCOUNT_KEY}")
             gcloud auth configure-docker us-docker.pkg.dev
+  quay-docker-login:
+    steps:
+      - run:
+          name: Login to Quay.io
+          command: echo "${QUAY_INFLUXDB3_PASSWORD}" |docker login --username="${QUAY_INFLUXDB3_USERNAME}" --password-stdin quay.io
 
 jobs:
   fmt:
@@ -437,11 +442,17 @@ jobs:
             export AWS_SECRET_ACCESS_KEY="${RELEASE_AWS_SECRET_ACCESS_KEY}"
             aws cloudfront create-invalidation --distribution-id "${RELEASE_ARTIFACTS_CLOUDFRONT}" --paths '/influxdb/<< parameters.destination >>/*'
   build-docker:
-    # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
-    # kinda small node)
+    parameters:
+      platform:
+        type: string
+      resource_class:
+        type: string
+      profile:
+        type: string
+        default: release
     machine:
       image: default
-    resource_class: 2xlarge+ # CPU bound, so make it fast
+    resource_class: << parameters.resource_class >>
     steps:
       - checkout
       - run:
@@ -449,17 +460,19 @@ jobs:
           command: |
             .circleci/scripts/docker_build_release.bash \
               "influxdb3" \
-              "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console" \
-              "influxdb3-edge:latest"
+              "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,system-py" \
+              "influxdb3:latest-<< parameters.platform>>" \
+              "<< parameters.platform >>" \
+              "<< parameters.profile >>"
 
           # linking might take a while and doesn't produce CLI output
-          no_output_timeout: 30m
+          no_output_timeout: 60m
       - run: |
-          docker save influxdb3-edge:latest >influxdb3-edge.tar
+          docker save influxdb3:latest-<<parameters.platform>> > influxdb3-<<parameters.platform>>.tar
       - persist_to_workspace:
           root: .
           paths:
-            - influxdb3-edge.tar
+            - influxdb3-<<parameters.platform>>.tar
   publish-docker:
     docker:
       - image: cimg/gcp:2023.02
@@ -467,12 +480,13 @@ jobs:
     steps:
       - checkout
       - setup_remote_docker
-      - gcloud-docker-login
+      - quay-docker-login
       - attach_workspace:
           at: .
       - run: |
-          docker load <influxdb3-edge.tar
-          .circleci/scripts/publish.bash influxdb3-edge
+          docker load <influxdb3-arm64.tar
+          docker load <influxdb3-amd64.tar
+          .circleci/scripts/publish.bash influxdb3 ${CIRCLE_SHA1}
 
   wait-for-docker:
     resource_class: small
@@ -601,13 +615,27 @@ workflows:
             - fmt
             - cargo-audit
       - build-docker:
-          <<: *release_filter
+          <<: *nofork_filter
+          name: build-docker-amd64
+          matrix:
+            parameters:
+              platform: [amd64]
+              resource_class: [2xlarge+]
+              profile: [quick-release]
+      - build-docker:
+          <<: *nofork_filter
+          name: build-docker-arm64
+          matrix:
+            parameters:
+              platform: [arm64]
+              resource_class: [arm.2xlarge]
+              profile: [quick-release]
       - publish-docker:
-          <<: *release_filter
           requires:
-            - build-docker
+            - build-docker-amd64
+            - build-docker-arm64
       - wait-for-docker:
-          <<: *release_filter
           requires:
-            - build-docker
-            - publish-docker
+            - build-docker-amd64
+            - build-docker-arm64
+            - publish-docker

.circleci/scripts/docker_build_release.bash

@@ -5,6 +5,8 @@ set -euo pipefail
 readonly PACKAGE="$1"
 readonly FEATURES="$2"
 readonly TAG="$3"
+readonly ARCH="${4:-amd64}"  # Default to amd64 if not specified
+readonly PROFILE="${5:-release}"       # Default to release if not specified
 
 RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)"
 COMMIT_SHA="$(git rev-parse HEAD)"
@@ -12,12 +14,17 @@ COMMIT_TS="$(env TZ=UTC0 git show --quiet --date='format-local:%Y-%m-%dT%H:%M:%S
 NOW="$(date --utc --iso-8601=seconds)"
 REPO_URL="https://github.com/influxdata/influxdb"
 
+# Convert arch to platform
+PLATFORM="linux/${ARCH}"
+
 exec docker buildx build \
   --build-arg CARGO_INCREMENTAL="no" \
   --build-arg CARGO_NET_GIT_FETCH_WITH_CLI="true" \
   --build-arg FEATURES="$FEATURES" \
   --build-arg RUST_VERSION="$RUST_VERSION" \
   --build-arg PACKAGE="$PACKAGE" \
+  --build-arg PROFILE="$PROFILE" \
+  --platform "$PLATFORM" \
   --label org.opencontainers.image.created="$NOW" \
   --label org.opencontainers.image.url="$REPO_URL" \
   --label org.opencontainers.image.revision="$COMMIT_SHA" \
@@ -28,4 +35,4 @@ exec docker buildx build \
   --label com.influxdata.image.package="$PACKAGE" \
   --progress plain \
   --tag "$TAG" \
-  .
+  .

.circleci/scripts/publish.bash

@@ -2,15 +2,31 @@
 set -euo pipefail
 
 release() {
-  image_src="${1}:latest"
-  image_dst="us-docker.pkg.dev/influxdata-team-edge/influxdb3-edge/${1}:${2}"
+  # This is a raw name, e.g. influxdb3
+  image_name="${1}"
+  image_dst="quay.io/influxdb/${1}:${2}"
 
-  if docker pull "${image_dst}" ; then
-    echo "docker image ${image_dst} already exists"
-    exit 0
-  fi
-  docker tag  "${image_src}" "${image_dst}"
-  docker push "${image_dst}"
+
+  # Tag images for each architecture
+  docker tag  "${image_name}:latest-amd64" "${image_dst}-amd64"
+  docker tag  "${image_name}:latest-arm64" "${image_dst}-arm64"
+
+  # push images for each architecture
+  docker push "${image_dst}-amd64"
+  docker push "${image_dst}-arm64"
+
+  docker manifest create "${image_dst}" \
+    --amend "${image_dst}-amd64" \
+    --amend "${image_dst}-arm64"
+
+  # Annotate the manifest with architecture and OS information
+  docker manifest annotate "${image_dst}" \
+    "${image_dst}-amd64" --arch amd64 --os linux
+  docker manifest annotate "${image_dst}" \
+    "${image_dst}-arm64" --arch arm64 --os linux
+
+  # Push up the manifest to create a multi-arch image.
+  docker manifest push "${image_dst}"
 }
 
 release "${1}" "${CIRCLE_SHA1}"

Dockerfile

@@ -1,12 +1,12 @@
 #syntax=docker/dockerfile:1.2
-ARG RUST_VERSION=1.75
+ARG RUST_VERSION=1.84
 FROM rust:${RUST_VERSION}-slim-bookworm as build
 
 # cache mounts below may already exist and owned by root
 USER root
 
 RUN apt update \
-    && apt install --yes binutils build-essential pkg-config libssl-dev clang lld git protobuf-compiler \
+    && apt install --yes binutils build-essential pkg-config libssl-dev clang lld git protobuf-compiler python3 python3-dev python3-pip \
     && rm -rf /var/lib/{apt,dpkg,cache,log}
 
 # Build influxdb3
@@ -16,7 +16,7 @@ WORKDIR /influxdb3
 ARG CARGO_INCREMENTAL=yes
 ARG CARGO_NET_GIT_FETCH_WITH_CLI=false
 ARG PROFILE=release
-ARG FEATURES=aws,gcp,azure,jemalloc_replacing_malloc
+ARG FEATURES=aws,gcp,azure,jemalloc_replacing_malloc,system-py
 ARG PACKAGE=influxdb3
 ENV CARGO_INCREMENTAL=$CARGO_INCREMENTAL \
     CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI \
@@ -39,7 +39,7 @@ RUN \
 FROM debian:bookworm-slim
 
 RUN apt update \
-    && apt install --yes ca-certificates gettext-base libssl3 --no-install-recommends \
+    && apt install --yes ca-certificates gettext-base libssl3 python3 python3-dev python3-pip --no-install-recommends \
     && rm -rf /var/lib/{apt,dpkg,cache,log} \
     && groupadd --gid 1500 influxdb3 \
     && useradd --uid 1500 --gid influxdb3 --shell /bin/bash --create-home influxdb3