ci: move some circleci tasks to self-hosted runners (#26514)

* ci: move some circleci tasks to self-hosted runners

we have self-hosted circleci runners. migrating to them will reduce the
cost dramatically. this only moves `machine:` jobs. work needs to be
done on the hosts before migrating the `docker:` jobs

* test(ci): change some filters to run jobs that otherwise wouldn't run

in order to test them on the self-hosted runners

if / when they pass, this commit needs to be dropped before merging

* ci: cleanup package-validation, run verification in containers

run the package validation scripts in containers on the self-hosted
runners. this has the benefit of not needing terraform, and also
prevents issues cleaning up the install on the long-lived runners by
using an ephemeral container for the installation

* ci: reset filters

several filters were changed for testing. this puts them back to their
original values

.circleci/config.yml

@@ -36,7 +36,6 @@ version: 2.1
 
 orbs:
   aws-s3:    circleci/aws-s3@2.0.0
-  terraform: circleci/terraform@2.1.0
   rust: circleci/rust@1.6.1
 
 # Unlike when a commit is pushed to a branch, CircleCI does not automatically
@@ -279,9 +278,8 @@ jobs:
 
   # Fetch python-build-standalone for official builds
   fetch-python:
-    machine:
+    machine: true
-      image: ubuntu-2204:current
+    resource_class: runner-ns/clustered-linux-vm
-      resource_class: medium
     environment:
       <<: *pbs_config
     steps:
@@ -501,9 +499,8 @@ jobs:
           path: artifacts/
 
   check_package_deb_amd64:
-    machine:
+    machine: true
-      image: ubuntu-2204:current
+    resource_class: runner-ns/clustered-linux-vm
-      resource_class: medium
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -511,13 +508,12 @@ jobs:
       - run:
           name: Validate Debian Package (AMD64)
           command: |
-            sudo .circleci/scripts/package-validation/debian \
+            docker run --rm --platform linux/amd64 -v /tmp/workspace:/tmp/workspace -v $(pwd):/repo -w /repo ubuntu:latest \
-              /tmp/workspace/artifacts/influxdb3*amd64.deb
+            .circleci/scripts/validate deb /tmp/workspace/artifacts/influxdb3*amd64.deb
 
   check_package_deb_arm64:
-    machine:
+    machine: true
-      image: ubuntu-2204:current
+    resource_class: runner-ns/clustered-linux-vm-arm
-      resource_class: arm.medium
     steps:
       - attach_workspace:
           at: /tmp/workspace
@@ -525,27 +521,30 @@ jobs:
       - run:
           name: Validate Debian Package (ARM64)
           command: |
-            sudo .circleci/scripts/package-validation/debian \
+            docker run --rm --platform linux/arm64 -v /tmp/workspace:/tmp/workspace -v $(pwd):/repo -w /repo ubuntu:latest \
-              /tmp/workspace/artifacts/influxdb3*arm64.deb
+            .circleci/scripts/validate deb /tmp/workspace/artifacts/influxdb3*arm64.deb
 
-  check_package_rpm:
+  check_package_rpm_amd64:
-    machine:
+    machine: true
-      image: ubuntu-2204:current
+    resource_class: runner-ns/clustered-linux-vm
-      resource_class: arm.medium
-    parameters:
-      arch:
-        type: string
     steps:
       - attach_workspace:
           at: /tmp/workspace
-      - add_ssh_keys:
-          fingerprints:
-            - 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67
       - checkout
       - run: |
-          AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID \
+          docker run --rm --platform linux/amd64 -v /tmp/workspace:/tmp/workspace -v $(pwd):/repo -w /repo redhat/ubi10 \
-          AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY \
+          .circleci/scripts/validate rpm /tmp/workspace/artifacts/influxdb3*x86_64.rpm
-          .circleci/scripts/package-validation/redhat << parameters.arch >> /tmp/workspace/artifacts/influxdb3*.<< parameters.arch >>.rpm
+
+  check_package_rpm_arm64:
+    machine: true
+    resource_class: runner-ns/clustered-linux-vm-arm
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - run: |
+          docker run --rm --platform linux/arm64 -v /tmp/workspace:/tmp/workspace -v $(pwd):/repo -w /repo redhat/ubi10 \
+          .circleci/scripts/validate rpm /tmp/workspace/artifacts/influxdb3*aarch64.rpm
 
   sign-packages:
     circleci_ip_ranges: true
@@ -633,8 +632,7 @@ jobs:
         default: influxdb3-core
     environment:
       <<: *pbs_config
-    machine:
+    machine: true
-      image: default
     resource_class: << parameters.resource_class >>
     steps:
       - checkout
@@ -787,20 +785,24 @@ workflows:
           <<: *release_filter
           requires:
             - build-packages
-      - check_package_rpm:
+      - check_package_rpm_arm64:
           <<: *nofork_filter
           name:
-            check_package_rpm-<< matrix.arch >>
+            check_package_rpm_arm64
-          matrix:
+          requires:
-            parameters:
+            - build-packages
-              arch: [ x86_64, aarch64 ]
+      - check_package_rpm_amd64:
+          <<: *nofork_filter
+          name:
+            check_package_rpm_amd64
           requires:
             - build-packages
       - sign-packages:
           <<: *release_filter
           requires:
             - build-packages
-            - check_package_rpm
+            - check_package_rpm_arm64
+            - check_package_rpm_amd64
             - check_package_deb_arm64
             - check_package_deb_amd64
             - test
@@ -821,12 +823,12 @@ workflows:
           <<: *docker_filter
           name: build-docker-amd64
           platform: amd64
-          resource_class: 2xlarge+
+          resource_class: runner-ns/clustered-linux-vm
       - build-docker:
           <<: *docker_filter
           name: build-docker-arm64
           platform: arm64
-          resource_class: arm.2xlarge
+          resource_class: runner-ns/clustered-linux-vm-arm
       - publish-docker:
           <<: *docker_filter
           requires:

.circleci/scripts/package-validation/debian

@@ -1,8 +0,0 @@
-#!/bin/bash
-set -o errexit \
-    -o nounset \
-    -o pipefail
-
-path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
-
-"${path}/validate" deb "${1}"

.circleci/scripts/package-validation/redhat

@@ -1,97 +0,0 @@
-#!/bin/bash
-set -o errexit \
-    -o nounset \
-    -o pipefail
-
-# $1 -> architecture
-# $2 -> package path
-case ${1} in
-  x86_64)  arch=x86_64 ;;
-  aarch64) arch=arm64  ;;
-esac
-
-package="$(realpath "${2}")"
-
-path="$(dirname "$(realpath "${BASH_SOURCE[0]}")")"
-
-terraform_init() {
-  pushd "${path}/tf" &>/dev/null
-
-  # Unfortunately, CircleCI doesn't offer any RPM based machine images.
-  # This is required to test the functionality of the systemd services.
-  # (systemd doesn't run within docker containers). This will spawn a
-  # Amazon Linux instance in AWS.
-  terraform init
-  terraform apply                   \
-    -auto-approve                   \
-    -var "architecture=${1}"        \
-    -var "package_path=${2}"        \
-    -var "identifier=${CIRCLE_JOB}"
-
-  popd &>/dev/null
-}
-
-terraform_free() {
-  pushd "${path}/tf" &>/dev/null
-
-  terraform destroy                 \
-    -auto-approve                   \
-    -var "architecture=${1}"        \
-    -var "package_path=${2}"        \
-    -var "identifier=${CIRCLE_JOB}"
-
-  popd &>/dev/null
-}
-
-terraform_ip() {
-  pushd "${path}/tf" &>/dev/null
-
-  terraform output -raw node_ssh
-
-  popd &>/dev/null
-}
-
-
-# This ensures that the associated resources within AWS are released
-# upon exit or when encountering an error. This is setup before the
-# call to "terraform apply" so even partially initialized resources
-# are released.
-# shellcheck disable=SC2064
-trap "terraform_free \"${arch}\" \"${package}\"" \
-  SIGINT  \
-  SIGTERM \
-  ERR     \
-  EXIT
-
-function terraform_setup()
-{
-  # TODO(bnpfeife): remove this once the executor is updated.
-  #
-  # Unfortunately, terraform provided by the CircleCI executor is *terribly*
-  # out of date. Most Linux distributions are disabling "ssh-rsa" public key
-  # algorithms which this uses to remote into the ec2 instance . This
-  # installs the latest version of terraform.
-  #
-  # Addendum: the "terraform_version" CircleCI option is broken!
-sudo tee /etc/apt/sources.list.d/hashicorp.list <<EOF >/dev/null || true
-deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main
-EOF
-
-  curl -fL https://apt.releases.hashicorp.com/gpg | gpg --dearmor | \
-    sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg >/dev/null
-
-  export DEBIAN_FRONTEND=noninteractive
-  sudo -E apt-get update
-  sudo -E apt-get install --yes terraform
-}
-
-terraform_setup
-
-terraform_init "${arch}" "${package}"
-
-printf 'Setup complete! Testing %s... (this takes several minutes!)' "${1}"
-
-# Since terraform *just* created this instance, the host key is not
-# known. Therefore, we'll disable StrictHostKeyChecking so ssh does
-# not wait for user input.
-ssh -o 'StrictHostKeyChecking=no' "ec2-user@$(terraform_ip)" 'sudo ./validate rpm ./influxdb3.rpm'

.circleci/scripts/package-validation/tf/main.tf

@@ -1,114 +0,0 @@
-terraform {
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "~> 2.70"
-    }
-  }
-}
-
-variable "architecture" {
-  type = string
-}
-
-variable "identifier" {
-  type = string
-}
-
-variable "package_path" {
-  type = string
-}
-
-provider "aws" {
-  region = "us-east-1"
-}
-
-data "aws_ami" "test_ami" {
-  most_recent = true
-
-  filter {
-    name   = "name"
-    values = ["al20*-ami-20*"]
-  }
-  filter {
-    name   = "virtualization-type"
-    values = ["hvm"]
-  }
-  filter {
-    name   = "architecture"
-    values = [var.architecture]
-  }
-
-  owners = ["137112412989"]
-}
-
-resource "aws_security_group" "influxdb_test_package_sg" {
-  ingress {
-    description = "Allow ssh connection"
-    from_port   = 22
-    to_port     = 22
-    protocol    = "tcp"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-
-  egress {
-    description = "Allow all outgoing"
-    from_port   = 0
-    to_port     = 0
-    protocol    = "all"
-    cidr_blocks = ["0.0.0.0/0"]
-  }
-}
-
-resource "aws_instance" "test_instance" {
-  count                  = 1
-  ami                    = data.aws_ami.test_ami.id
-  instance_type          = var.architecture == "x86_64" ? "t2.micro" : "c6g.medium"
-  key_name               = "circleci-oss-test"
-  vpc_security_group_ids = [aws_security_group.influxdb_test_package_sg.id]
-
-  tags = {
-    Name = format("circleci_%s_test_%s", var.identifier, var.architecture)
-  }
-
-  provisioner "file" {
-    source      = var.package_path
-    destination = "/home/ec2-user/influxdb3.rpm"
-
-    connection {
-      type     = "ssh"
-      user     = "ec2-user"
-      host     = self.public_dns
-      agent    = true
-    }
-  }
-
-  provisioner "file" {
-    source      = "../validate"
-    destination = "/home/ec2-user/validate"
-
-    connection {
-      type     = "ssh"
-      user     = "ec2-user"
-      host     = self.public_dns
-      agent    = true
-    }
-  }
-
-  provisioner "remote-exec" {
-    inline = [
-      "chmod +x /home/ec2-user/validate",
-    ]
-
-    connection {
-      type     = "ssh"
-      user     = "ec2-user"
-      host     = self.public_dns
-      agent    = true
-    }
-  }
-}
-
-output "node_ssh" {
-  value = aws_instance.test_instance.0.public_dns
-}