build: replace local e2e tests with e2e-monitor-ci (#21387)

* feat: update e2e to use monitor-ci * build: local copy of test script * build: use stable docker image for e2e * fix: use a working image for e2e ci job
2021-05-07 14:47:56 -04:00 · 2021-05-07 14:47:56 -04:00 · dc72dc3f0c
parent 883241eadb
commit dc72dc3f0c
2 changed files with 276 additions and 61 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -29,7 +29,7 @@ workflows:
      - cross_build:
          requires:
            - godeps
-      - e2e:
+      - e2e-monitor-ci:
          requires:
            - build
      - grace_daily:
@ -50,7 +50,7 @@ workflows:
              only:
                - master
          requires:
-            - e2e
+            - e2e-monitor-ci

  nightly:
    triggers:
@ -98,10 +98,6 @@ workflows:
          requires:
            - deploy_nightly

-orbs:
-  # Needed to install chrome for e2e testing.
-  browser-tools: circleci/browser-tools@1.1
-
 commands:
  # Install system dependencies needed to run a native build of influxd
  install_core_deps:
@ -143,9 +139,7 @@ commands:
                command: |
                  echo -e "$GPG_PRIVATE_KEY" > private.key
                  gpg --batch --import private.key
-            - run:
-                name: Log into Quay
-                command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
+            - quay_login
      - run:
          name: Set GOPATH
          # Machine executors use a different GOPATH from the cimg/go Docker executors.
@ -277,6 +271,25 @@ commands:
            - run:
                name: Publish release
                command: goreleaser --debug release -p 1 --rm-dist --skip-validate
+  quay_login:
+    steps:
+      - run:
+          name: Log in to Quay
+          command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
+
+  bail_if_forked:
+    parameters:
+      exit_code:
+        type: integer
+        default: 1
+    steps:
+      - run:
+          name: Exit if running from a fork
+          command: |
+            if [ -n "$CIRCLE_PR_NUMBER" ]; then
+              echo 'Error: Running from a fork, exiting!'
+              exit << parameters.exit_code >>
+            fi

 jobs:
  #########################
@ -501,6 +514,17 @@ jobs:
            - /tmp/go-cache
      - store_artifacts:
          path: bin/linux
+      - persist_to_workspace:
+          root: .
+          paths:
+            - project
+            - bin/linux/influxd
+            - bin/linux/influx
+            - etc/litmus_success_notify.sh
+            - etc/litmus_fail_notify.sh
+      # if we are running on a fork, exit with success now instead of building & pushing a docker image.
+      - bail_if_forked:
+          exit_code: 0
      - setup_remote_docker:
          version: 19.03.13
          docker_layer_caching: true
@ -513,12 +537,11 @@ jobs:
      - persist_to_workspace:
          root: .
          paths:
-            - project
-            - bin/linux/influxd
-            - bin/linux/influx
-            - etc/litmus_success_notify.sh
-            - etc/litmus_fail_notify.sh
            - docker-image.tar
+      - quay_login
+      - run:
+          name: Push the candidate image to quay
+          command: docker push quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1}

  cross_build:
    machine:
@ -591,54 +614,25 @@ jobs:
  ### e2e/integration test jobs ###
  #################################

-  e2e:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    environment:
-      GOCACHE: /tmp/go-cache
-    working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
+  e2e-monitor-ci:
+    docker:
+      - image: cimg/go:1.15.6
    steps:
-      - browser-tools/install-browser-tools
+      # if we are running from a fork, exit with a fail status.
+      - bail_if_forked
      - checkout
-      - attach_workspace:
-          at: /home/circleci/go/src/github.com/influxdata/influxdb
-      - restore_cache:
-          name: Restore Yarn Cache
-          keys:
-            - yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
-      - run: sudo apt-get update && sudo apt-get install netcat-openbsd
      - run:
-          name: Install Dependencies
-          command: |
-            set +e
-            cd ui
-            yarn install --immutable
-      - save_cache:
-          name: Save Yarn Cache
-          key: yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
-          paths:
-            - /home/circleci/go/src/github.com/influxdata/influxdb/ui/node_modules
-            - ~/.cache/yarn
-            - ~/.cache/Cypress
-      - run:
-          name: Start the OSS docker image
-          command: |
-            docker load < docker-image.tar
-            docker run -p 8086:8086 \
-              -e INFLUXD_STORE=memory \
-              -e INFLUXD_E2E_TESTING=true \
-              quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1}
-          background: true
-      - run: make e2e
-      - store_test_results:
-          path: ui/junit-results
+          name: Run the tests
+          command: API_KEY=${MONITOR_CI_API_KEY} MONITOR_CI_BRANCH="master" OSS_SHA=${CIRCLE_SHA1} ./scripts/ci/run-monitor-ci-tests.bash
      - store_artifacts:
-          path: ui/cypress/videos
-          destination: videos
+          path: monitor-ci/test-artifacts/results/build-oss-image
+          destination: test_artifacts/results/build-oss-image
      - store_artifacts:
-          path: ui/cypress/screenshots
-          destination: screenshots
+          path: monitor-ci/test-artifacts/results/oss-e2e
+          destination: test_artifacts/results/oss-e2e
+      - store_artifacts:
+          path: monitor-ci/test-artifacts/results/shared
+          destination: test_artifacts/results/shared

  litmus_daily:
    machine: true
@ -735,13 +729,10 @@ jobs:
    steps:
      - attach_workspace:
          at: /home/circleci/go/src/github.com/influxdata/influxdb
-      - run:
-          name: Log into Quay
-          command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
+      - quay_login
      - run:
          name: Push the image to Quay
          command: |
            docker load < docker-image.tar
            docker tag quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1} quay.io/influxdb/oss-acceptance:latest
-            docker push quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1}
            docker push quay.io/influxdb/oss-acceptance:latest
--- a/scripts/ci/run-monitor-ci-tests.bash
+++ b/scripts/ci/run-monitor-ci-tests.bash
@ -0,0 +1,224 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+########################
+# --- Script Summary ---
+# This script is the junction between the CIs of the public UI and influxdb OSS repos and the monitor-ci CI (private).
+# When the public CI is started, this script kicks off the private CI and waits for it to complete.
+# This script uses the CircleCI APIs to make this magic happen.
+#
+# If the private CI fails, this script will collect the names and artifacts of the failed jobs and report them.
+# This script should support multiple workflows if more are added, although it has not been tested.
+# This script waits 40 minutes for the private CI to complete otherwise it fails.
+#
+# **For Running from the UI Repository:**
+# If you retry a failing job in the private CI and it passes, you can safely rerun this job in the public CI.
+#  - This script uses your commit SHA to search for a passing pipeline before starting a new one.
+#  - If you rerun the private CI and it passes, this script will find that pipeline and will not start a new one.
+#  - In this situation the script will exit quickly with success.
+#
+# Required Env Vars for Running from the UI Repository:
+# - SHA: the UI repo commit SHA we're running against
+# - API_KEY: the CircleCI API access key
+# - UI_BRANCH: the branch of the UI repo we're running against
+# - MONITOR_CI_BRANCH: the branch of the monitor-ci repo to start a pipeline with (usually 'master')
+# - PULL_REQUEST: the open pull request, if one exists (used for lighthouse)
+#
+# **For running from the influxdb OSS repository:**
+# Since the OSS private CI is very simple, retrying a failing job in the private CI is not supported.
+# Currently, only running e2e tests against the latest from the UI master branch is supported. 
+#
+# Required Env Vars for Running from the influxdb OSS Repository:
+# - OSS_SHA: the influxdb repo commit SHA we're running against
+# - MONITOR_CI_BRANCH: the branch of the monitor-ci repo to start a pipeline with (usually 'master')
+########################
+
+# make dir for artifacts
+mkdir -p monitor-ci/test-artifacts/results/{build-oss-image,oss-e2e,build-image,cloud-e2e,cloud-e2e-firefox,cloud-e2e-k8s-idpe,cloud-lighthouse,smoke,build-prod-image,deploy}/{shared,oss,cloud}
+# if we are not running from the OSS repo, OSS_SHA will be unset, and we must be running from the UI repo.
+if [[ -z "${OSS_SHA:-}" ]]; then
+	# get monitor-ci pipelines we've already run on this SHA
+	found_passing_pipeline=0
+	all_pipelines=$(curl -s --request GET \
+			--url "https://circleci.com/api/v2/project/gh/influxdata/monitor-ci/pipeline" \
+			--header "Circle-Token: ${API_KEY}" \
+			--header 'content-type: application/json' \
+			--header 'Accept: application/json')
+
+	# check the status of the workflows for each of these pipelines
+	all_pipelines_ids=( $(echo ${all_pipelines} | jq -r '.items | .[].id') )
+	for pipeline_id in "${all_pipelines_ids[@]}"; do
+
+		config=$(curl -s --request GET \
+			--url "https://circleci.com/api/v2/pipeline/${pipeline_id}/config" \
+			--header "Circle-Token: ${API_KEY}" \
+			--header 'content-type: application/json' \
+			--header 'Accept: application/json')
+
+		# finds the UI SHA parameter used in this pipeline by hunting for the line "export UI_SHA="
+		pipeline_ui_sha=$(echo ${config} | jq '.compiled' | grep -o 'export UI_SHA=[^\]*' | grep -v 'export UI_SHA=${LATEST_SHA}' | head -1 | sed 's/=/\n/g' | tail -1 || true)
+
+		if [[ "${SHA}" == "${pipeline_ui_sha}" ]]; then
+			# check if this pipeline's 'build' workflow is passing
+			workflows=$(curl -s --request GET \
+				--url "https://circleci.com/api/v2/pipeline/${pipeline_id}/workflow" \
+				--header "Circle-Token: ${API_KEY}" \
+				--header 'content-type: application/json' \
+				--header 'Accept: application/json')
+
+			number_build_success_workflows=$(echo ${workflows} | jq '.items | map(select(.name == "build" and .status == "success")) | length')
+			if [ $number_build_success_workflows -gt 0 ]; then
+				# we've found a successful run
+				found_passing_pipeline=1
+				break
+			fi
+		fi
+	done
+
+	# terminate early if we found a passing pipeline for this SHA
+	if [ $found_passing_pipeline -eq 1 ]; then
+		printf "\nSUCCESS: Found a passing monitor-ci pipeline for this SHA, will not re-run these tests\n"
+		exit 0
+	else
+		printf "\nno passing monitor-ci pipelines found for this SHA, starting a new one\n"
+	fi
+
+	# set the parameters for starting the monitor-ci pipeline from the UI repo
+	DEPLOY_PROD=false
+	if [[ "${UI_BRANCH}" == "master" ]]; then
+		DEPLOY_PROD=false # TODO: change this to true when we're ready to depend on this script
+	fi
+	pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH}, UI branch ${UI_BRANCH} and using UI SHA ${SHA}"
+	reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"ui-sha\":\"${SHA}\", \"ui-branch\":\"${UI_BRANCH}\", \"ui-pull-request\":\"${PULL_REQUEST}\", \"deploy-prod\":${DEPLOY_PROD}}}"
+else
+	# set the parameters for starting the monitor-ci pipeline from the influxdb repo
+	pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH} using OSS SHA ${OSS_SHA}"
+	reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"oss-sha\":\"${OSS_SHA}\" }}"
+fi
+
+printf "\n${pipelineStartMsg}\n"
+pipeline=$(curl -s --fail --request POST \
+	--url https://circleci.com/api/v2/project/gh/influxdata/monitor-ci/pipeline \
+	--header "Circle-Token: ${API_KEY}" \
+	--header 'content-type: application/json' \
+	--header 'Accept: application/json'    \
+	--data "${reqData}")
+
+if [ $? != 0 ]; then
+	echo "failed to start the monitor-ci pipeline, quitting"
+	exit 1
+fi
+
+pipeline_id=$(echo ${pipeline} | jq  -r '.id')
+pipeline_number=$(echo ${pipeline} | jq -r '.number')
+
+printf "\nwaiting for monitor-ci pipeline to begin...\n"
+sleep 1m
+printf "\nmonitor-ci pipeline has begun. Running pipeline number ${pipeline_number} with id ${pipeline_id}\n"
+
+# poll the status of the monitor-ci pipeline
+is_failure=0
+attempts=0
+max_attempts=40 # minutes
+while [ $attempts -le $max_attempts ];
+do
+
+	workflows=$(curl -s --request GET \
+		--url "https://circleci.com/api/v2/pipeline/${pipeline_id}/workflow" \
+		--header "Circle-Token: ${API_KEY}" \
+		--header 'content-type: application/json' \
+		--header 'Accept: application/json')
+
+	number_running_workflows=$(echo ${workflows} | jq  -r '.items | map(select(.status == "running" or .status == "failing")) | length')
+
+	# when the pipeline has finished
+	if [ ${number_running_workflows} -eq 0 ]; then
+		workflows_ids=( $(echo ${workflows} | jq -r '.items | .[].id') )
+
+		# report failed jobs per workflow
+		for workflow_id in "${workflows_ids[@]}"; do
+			workflow_status=$(echo ${workflows} | jq -r --arg id "${workflow_id}" '.items | map(select(.id == $id)) | .[].status')
+
+			if [[ "$workflow_status" == "success" ]]; then
+				printf "\nSUCCESS: monitor-ci workflow with id ${workflow_id} passed: https://app.circleci.com/pipelines/github/influxdata/monitor-ci/${pipeline_number}/workflows/${workflow_id} \n"
+			else
+				# set job failure
+				is_failure=1
+
+				# get the jobs that failed for this workflow
+				jobs=$(curl -s --request GET \
+					--url "https://circleci.com/api/v2/workflow/${workflow_id}/job" \
+					--header "Circle-Token: ${API_KEY}" \
+					--header 'content-type: application/json' \
+					--header 'Accept: application/json')
+
+				# print the names of the failed jobs
+				printf "\nFailed jobs:\n"
+				failed_jobs=$(echo ${jobs} | jq '.items | map(select(.status == "failed"))')
+				failed_jobs_names=( $(echo ${failed_jobs} | jq -r '.[].name') )
+				for name in "${failed_jobs_names[@]}"; do
+					printf " - ${name}\n"
+				done
+
+				# get the artifacts for each failed job
+				printf "\nArtifacts from failed jobs:\n"
+				for name in "${failed_jobs_names[@]}"; do
+					printf "\n===== ${name} =====\n"
+					job_number=$(echo ${failed_jobs} | jq -r --arg name "${name}" 'map(select(.name == $name)) | .[].job_number')
+					artifacts=$(curl -s --request GET \
+						--url "https://circleci.com/api/v1.1/project/github/influxdata/monitor-ci/${job_number}/artifacts" \
+						--header "Circle-Token: ${API_KEY}" \
+						--header 'content-type: application/json' \
+						--header 'Accept: application/json')
+
+					artifacts_length=$(echo ${artifacts} | jq -r 'length')
+					if [ ${artifacts_length} -eq 0 ]; then
+						printf "\n No artifacts for this failed job.\n"
+					else
+						artifacts_urls=( $(echo ${artifacts} | jq -r '.[].url') )
+						# download each artifact
+						for url in "${artifacts_urls[@]}"; do
+							path=$(echo ${artifacts} | jq --arg url "${url}" 'map(select(.url == $url)) | .[].pretty_path')
+
+							# download artifact
+							filename=$(basename "${path}")
+							filename="${filename::-1}" # removes extra " from end
+							# put shared artifacts in the shared folder
+							if [[ "${path}" == *"shared"* ]] ; then
+								subdirectory="shared"
+							else
+								if [[ "${path}" == *"cloud"* ]] ; then
+									subdirectory="cloud"
+								else
+									subdirectory="oss"
+								fi
+							fi
+							output="monitor-ci/test-artifacts/results/${name}/${subdirectory}/${filename}"
+							curl -L -s --request GET \
+								--output "${output}" \
+								--url "${url}" \
+								--header "Circle-Token: ${API_KEY}"
+						done
+						printf "\n ${artifacts_length} artifacts successfully downloaded for this failed job.\n"
+					fi
+				done
+
+				printf "\n\nFAILURE: monitor-ci workflow with id ${workflow_id} failed:\n"
+				printf "https://app.circleci.com/pipelines/github/influxdata/monitor-ci/${pipeline_number}/workflows/${workflow_id}"
+			fi
+		done
+
+		exit $is_failure
+	fi
+
+	# sleep 1 minute and poll the status again
+	attempts=$(($attempts+1))
+	remaining_attempts=$(($max_attempts-$attempts))
+	printf "\nmonitor-ci pipeline ${pipeline_number} isn't finished yet, waiting another minute... ($remaining_attempts minutes left)\n"
+	sleep 1m
+
+done
+
+printf "\nmonitor-ci pipeline did not finish in time, quitting\n"
+exit 1