fix: systemd service -- handle 40x and block indefinitely (#22174)

* fix: systemd service -- handle 40x and block indefinitely

* chore: update changelog

* chore: add pkg_run_test to CI

* chore: fix curl syntax for auth setup in pkg_run_test

.circleci/config.yml

@@ -1,5 +1,8 @@
 version: "2.1"
 
+orbs:
+  terraform: circleci/terraform@2.1.0
+
 parameters:
   aws_teardown:
     default: false
@@ -48,6 +51,9 @@ workflows:
               ignore: /pull\/[0-9]+/
           requires:
             - build
+      - pkg_run_test:
+          requires:
+            - cross_build
       - perf_test:
           name: perf-test-flux
           format: flux-http
@@ -556,6 +562,57 @@ jobs:
           paths:
             - artifacts/*amd64.deb
 
+  pkg_run_test:
+    executor: terraform/default
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - add_ssh_keys:
+          fingerprints:
+            - "91:0a:5b:a7:f9:46:77:f3:5d:4a:cf:d2:44:c8:2c:5a"
+      - terraform/validate:
+          path: scripts/ci/
+      - run:
+          name: Terraform apply
+          command: |
+            set -x
+            export DEBNAME="/tmp/workspace/artifacts/influxdb2-nightly-amd64.deb"
+            terraform -chdir=scripts/ci init -input=false
+            AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
+              -chdir=scripts/ci \
+              apply \
+                -auto-approve \
+                -var package_path=${DEBNAME} \
+                -var instance_name=circleci-terraform-${CIRCLE_SHA1} \
+                -var additional_files_dir=${PWD}/scripts/ci/tests/
+      - run:
+          name: Install deb
+          command: |
+            set -x
+            export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh)
+            ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip \<< EOF
+            sudo apt-get update && sudo apt-get install -y /home/ubuntu/influxdb.deb
+            EOF
+      - run:
+          name: Run tests
+          command: |
+            set -x
+            export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh)
+            files=$(ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip 'find /home/ubuntu/files/ -maxdepth 1 -mindepth 1 | sort')
+            for file in $files; do
+              ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip "sudo $file"
+            done
+      - run:
+          name: Terraform destroy
+          when: always
+          command: |
+            AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
+              -chdir=scripts/ci \
+              destroy \
+                -auto-approve
+
+
   perf_test:
     machine:
       image: ubuntu-2004:202010-01

CHANGELOG.md

@@ -79,6 +79,7 @@ This release adds an embedded SQLite database for storing metadata required by t
 1. [21962](https://github.com/influxdata/influxdb/pull/21962): Flux metaqueries for `_field` take fast path if `_measurement` is the only predicate.
 1. [22059](https://github.com/influxdata/influxdb/pull/22059): Copy names from mmapped memory before closing iterator
 1. [22186](https://github.com/influxdata/influxdb/pull/22186): Preserve comments in flux queries when saving task definitions
+1. [#22174](https://github.com/influxdata/influxdb/pull/22174): systemd service -- handle 40x and block indefinitely
 
 ## v2.0.7 [2021-06-04]
 

scripts/ci/main.tf

@@ -0,0 +1,152 @@
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~> 2.70"
+    }
+  }
+}
+
+####################
+# Declare variables
+
+# Variables without default values
+# these variables need to be changed - see terraform.tfvars
+variable "key_name" { }
+variable "package_path" { }
+variable "instance_name" { }
+
+# Variables with default values
+variable "additional_files_dir" {
+  type = string
+  default = ""
+}
+
+variable "instance_type" {
+  type = string
+  default = "t3.micro"
+}
+
+variable "region" {
+  type = string
+  default = "us-west-2"
+}
+
+####################
+# Declare data
+locals {
+  additional_files_dest = "/home/ubuntu/files"
+  package_path = "/tmp/workspace/packages"
+  ubuntu_home = "/home/ubuntu"
+  ubuntu_user = "ubuntu"
+}
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+  }
+
+  filter {
+    name   = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  owners = ["099720109477"] # Canonical
+}
+
+####################
+# Declare resources
+provider "aws" {
+  profile = "default"
+  region  = var.region
+}
+
+# The security group defines access restrictions
+resource "aws_security_group" "influxdb_test_sg" {
+  ingress {
+    description = "Allow ssh connection"
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    description = "Allow all egress"
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+}
+
+# The data node for the cluster
+resource "aws_instance" "test_node" {
+  count = 1
+
+  ami           = data.aws_ami.ubuntu.id
+  instance_type = var.instance_type
+  key_name = var.key_name
+  vpc_security_group_ids = [aws_security_group.influxdb_test_sg.id]
+
+  tags = {
+    Name = var.instance_name
+  }
+
+  provisioner "file" {
+    source = var.package_path
+    destination = "${local.ubuntu_home}/influxdb.deb"
+
+    connection {
+      type     = "ssh"
+      user     = local.ubuntu_user
+      host     = self.public_dns
+      agent    = true
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "mkdir -p ${local.additional_files_dest}",
+    ]
+
+    connection {
+      type     = "ssh"
+      user     = local.ubuntu_user
+      host     = self.public_dns
+      agent    = true
+    }
+  }
+
+  provisioner "file" {
+    source = var.additional_files_dir
+    destination = "${local.additional_files_dest}"
+
+    connection {
+      type     = "ssh"
+      user     = local.ubuntu_user
+      host     = self.public_dns
+      agent    = true
+    }
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "chmod +x ${local.additional_files_dest}/*.sh",
+    ]
+
+    connection {
+      type     = "ssh"
+      user     = local.ubuntu_user
+      host     = self.public_dns
+      agent    = true
+    }
+  }
+}
+
+####################
+# Declare outputs
+output "test_node_ssh" { value = aws_instance.test_node.0.public_dns }

scripts/ci/terraform.tfvars

@@ -0,0 +1,6 @@
+##################################
+# YOU MUST CHANGE THESE VARIABLES
+
+# find your key pair id (or create one) at https://console.aws.amazon.com/ec2/v2/home#KeyPairs
+# You will need your private key to ssh to your instances
+key_name = "circleci-oss-test"

scripts/ci/tests/01-default-config.sh

@@ -0,0 +1,4 @@
+#!/bin/bash -eux
+
+service influxdb start
+service influxdb stop

scripts/ci/tests/02-self-signed-certs.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eux
+
+echo 'tls-cert = "/etc/ssl/influxdb.crt"' >> /etc/influxdb/config.toml
+echo 'tls-key = "/etc/ssl/influxdb.key"' >> /etc/influxdb/config.toml
+openssl req -x509 -nodes -newkey rsa:2048 -keyout /etc/ssl/influxdb.key -out /etc/ssl/influxdb.crt -days 365 -subj /C=US/ST=CA/L=sanfrancisco/O=influxdata/OU=edgeteam/CN=localhost
+chown influxdb:influxdb /etc/ssl/influxdb.*
+service influxdb start
+service influxdb stop
+contents="$(head -n -2 /etc/influxdb/config.toml)"
+echo "$contents" > /etc/influxdb/config.toml

scripts/ci/tests/03-auth-enabled.sh

@@ -0,0 +1,10 @@
+#!/bin/bash -eux
+
+service influxdb start
+result=$(curl -s -o /dev/null -H "Content-Type: application/json" -XPOST -d '{"username": "default", "password": "thisisnotused", "retentionPeriodSeconds": 0, "org": "testorg", "bucket": "unusedbucket", "token": "thisisatesttoken"}' http://localhost:8086/api/v2/setup -w %{http_code})
+if [ "$result" != "201" ]; then
+  exit 1
+fi
+service influxdb stop
+service influxdb start
+service influxdb stop

scripts/influxd-systemd-start.sh

@@ -1,17 +1,30 @@
 #!/bin/bash -e
 
 /usr/bin/influxd &
-echo $! > /var/lib/influxdb/influxd.pid
+PID=$!
+echo $PID > /var/lib/influxdb/influxd.pid
 
+PROTOCOL="http"
 BIND_ADDRESS=$(influxd print-config --key-name http-bind-address)
+TLS_CERT=$(influxd print-config --key-name tls-cert | tr -d '"')
+TLS_KEY=$(influxd print-config --key-name tls-key | tr -d '"')
+if [ -n "${TLS_CERT}" ] && [ -n "${TLS_KEY}" ]; then
+  echo "TLS cert and key found -- using https"
+  PROTOCOL="https"
+fi
 HOST=${BIND_ADDRESS%%:*}
 HOST=${HOST:-"localhost"}
 PORT=${BIND_ADDRESS##*:}
 
 set +e
-result=$(curl -s -o /dev/null http://$HOST:$PORT/ready -w %{http_code})
-while [ "$result" != "200" ]; do
+attempts=0
+url="$PROTOCOL://$HOST:$PORT/ready"
+result=$(curl -k -s -o /dev/null $url -w %{http_code})
+while [ "${result:0:2}" != "20" ] && [ "${result:0:2}" != "40" ]; do
+  attempts=$(($attempts+1))
+  echo "InfluxDB API at $url unavailable after $attempts attempts..."
   sleep 1
-  result=$(curl -s -o /dev/null http://$HOST:$PORT/ready -w %{http_code})
+  result=$(curl -k -s -o /dev/null $url -w %{http_code})
 done
+echo "InfluxDB started"
 set -e