build: update script to run ui e2e tests (#22670)
* build: update script to run ui e2e tests * chore: update comments for missing env var descriptionspull/22691/head
William Baker
GitHub
parent
2433e00e1c
commit
d8430aa7dd
|
|
@ -10,9 +10,12 @@ set -eu -o pipefail
|
|||
#
|
||||
# If the private CI fails, this script will collect the names and artifacts of the failed jobs and report them.
|
||||
# This script should support multiple workflows if more are added, although it has not been tested.
|
||||
# This script waits 40 minutes for the private CI to complete otherwise it fails.
|
||||
# This script waits 50 minutes for the private CI to complete otherwise it fails.
|
||||
#
|
||||
# **For Running from the UI Repository:**
|
||||
# If you want to retry failed jobs in the private CI, simply retry this job from the public CI.
|
||||
# - This script uses your commit SHA to search for a failed pipeline to retry before starting a new one.
|
||||
#
|
||||
# If you retry a failing job in the private CI and it passes, you can safely rerun this job in the public CI.
|
||||
# - This script uses your commit SHA to search for a passing pipeline before starting a new one.
|
||||
# - If you rerun the private CI and it passes, this script will find that pipeline and will not start a new one.
|
||||
|
|
@ -25,21 +28,93 @@ set -eu -o pipefail
|
|||
# - MONITOR_CI_BRANCH: the branch of the monitor-ci repo to start a pipeline with (usually 'master')
|
||||
# - PULL_REQUEST: the open pull request, if one exists (used for lighthouse)
|
||||
#
|
||||
# **For running from the influxdb OSS repository:**
|
||||
# **For OSS-specific testing:**
|
||||
# Since the OSS private CI is very simple, retrying a failing job in the private CI is not supported.
|
||||
# Currently, only running e2e tests against the latest from the UI master branch is supported.
|
||||
# OSS-specific testing can include evaluating changes on the OSS master branch against the latest UI acceptance image
|
||||
# to make sure OSS API changes don't break the UI, and evaluating changes to an OSS binary with embedded UI assets with
|
||||
# a specified UI commit that the UI is from (like a tagged release commit). This allows for the master branches of
|
||||
# both the UI and influxdb OSS respositories to always stay compatible, and for OSS release builds to be e2e tested
|
||||
# without needing to duplicate the entire private test infrastructure provided in monitor-ci.
|
||||
#
|
||||
# Required Env Vars for Running from the influxdb OSS Repository:
|
||||
# Required Env Vars for Testing Changes to OSS Master with the Latest Image Published from UI Master:
|
||||
# - API_KEY: See "Required Env Vars for Running from the UI Repository".
|
||||
# - OSS_SHA: the influxdb repo commit SHA we're running against
|
||||
# - MONITOR_CI_BRANCH: the branch of the monitor-ci repo to start a pipeline with (usually 'master')
|
||||
#
|
||||
# Required Env Vars for Testing Changes to an OSS Image with Embedded UI with e2e tests from a Specific UI Commit:
|
||||
# - API_KEY: See "Required Env Vars for Running from the UI Repository".
|
||||
# - SHA: the UI repo commit SHA we want to build and run e2e tests from
|
||||
# - UI_BRANCH: the branch of the UI repo we're running against
|
||||
# - OSS_SHA: the influxdb repo commit SHA we're running against
|
||||
# - MONITOR_CI_BRANCH: the branch of the monitor-ci repo to start a pipeline with (usually 'master')
|
||||
|
||||
########################
|
||||
|
||||
# starts a new monitor-ci pipeline with provided parameters
|
||||
startNewPipeline() {
|
||||
pipelineStartMsg=$1
|
||||
reqData=$2
|
||||
|
||||
printf "\n${pipelineStartMsg}\n"
|
||||
pipeline=$(curl -s --fail --request POST \
|
||||
--url https://circleci.com/api/v2/project/gh/influxdata/monitor-ci/pipeline \
|
||||
--header "Circle-Token: ${API_KEY}" \
|
||||
--header 'content-type: application/json' \
|
||||
--header 'Accept: application/json' \
|
||||
--data "${reqData}")
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "failed to start the monitor-ci pipeline, quitting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# set variables to identify pipeline to watch
|
||||
pipeline_id=$(echo ${pipeline} | jq -r '.id')
|
||||
pipeline_number=$(echo ${pipeline} | jq -r '.number')
|
||||
|
||||
printf "\nwaiting for monitor-ci pipeline to begin...\n"
|
||||
sleep 1m
|
||||
printf "\nmonitor-ci pipeline has begun. Running pipeline number ${pipeline_number} with id ${pipeline_id}\n"
|
||||
}
|
||||
|
||||
# retries all failed jobs from a previously failed monitor-ci pipeline
|
||||
retryFailedPipeline() {
|
||||
failed_pipeline_workflow_id=$1
|
||||
failed_pipeline_id=$2
|
||||
failed_pipeline_number=$3
|
||||
|
||||
pipeline=$(curl -s --fail --request POST \
|
||||
--url https://circleci.com/api/v2/workflow/${failed_pipeline_workflow_id}/rerun \
|
||||
--header "Circle-Token: ${API_KEY}" \
|
||||
--header 'content-type: application/json' \
|
||||
--header 'Accept: application/json' \
|
||||
--data "{ \"from_failed\": true }")
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "failed to re-run the monitor-ci pipeline, quitting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# set variables to identify pipeline to watch
|
||||
pipeline_id=$failed_pipeline_id
|
||||
pipeline_number=$failed_pipeline_number
|
||||
|
||||
printf "\nwaiting for monitor-ci pipeline to begin the re-run...\n"
|
||||
sleep 1m
|
||||
printf "\nmonitor-ci pipeline re-run has begun. Running pipeline number ${pipeline_number} with id ${pipeline_id}\n"
|
||||
}
|
||||
|
||||
# make dir for artifacts
|
||||
mkdir -p monitor-ci/test-artifacts/results/{build-oss-image,oss-e2e,build-image,cloud-e2e,cloud-e2e-firefox,cloud-e2e-k8s-idpe,cloud-lighthouse,smoke,build-prod-image,deploy}/{shared,oss,cloud}
|
||||
# if we are not running from the OSS repo, OSS_SHA will be unset, and we must be running from the UI repo.
|
||||
|
||||
# get monitor-ci pipelines we've already run on this SHA
|
||||
found_passing_pipeline=0
|
||||
found_failed_pipeline=0
|
||||
|
||||
# If OSS_SHA is not set, run the standard UI pipeline with the latest OSS backend docker image.
|
||||
if [[ -z "${OSS_SHA:-}" ]]; then
|
||||
# get monitor-ci pipelines we've already run on this SHA
|
||||
found_passing_pipeline=0
|
||||
required_workflows=( "build" )
|
||||
|
||||
all_pipelines=$(curl -s --request GET \
|
||||
--url "https://circleci.com/api/v2/project/gh/influxdata/monitor-ci/pipeline" \
|
||||
--header "Circle-Token: ${API_KEY}" \
|
||||
|
|
@ -73,6 +148,16 @@ if [[ -z "${OSS_SHA:-}" ]]; then
|
|||
found_passing_pipeline=1
|
||||
break
|
||||
fi
|
||||
|
||||
number_build_failed_workflows=$(echo ${workflows} | jq '.items | map(select(.name == "build" and .status == "failed")) | length')
|
||||
if [ $number_build_failed_workflows -gt 0 ]; then
|
||||
# there's a failed run, let's retry it
|
||||
found_failed_pipeline=1
|
||||
failed_pipeline_workflow_id=$(echo ${workflows} | jq -r '.items | .[0] | .id')
|
||||
failed_pipeline_id=$pipeline_id
|
||||
failed_pipeline_number=$(echo ${all_pipelines} | jq -r --arg pipeline_id "${pipeline_id}" '.items | map(select(.id == $pipeline_id)) | .[0] | .number')
|
||||
break
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
@ -80,6 +165,8 @@ if [[ -z "${OSS_SHA:-}" ]]; then
|
|||
if [ $found_passing_pipeline -eq 1 ]; then
|
||||
printf "\nSUCCESS: Found a passing monitor-ci pipeline for this SHA, will not re-run these tests\n"
|
||||
exit 0
|
||||
elif [ $found_failed_pipeline -eq 1 ]; then
|
||||
printf "\nfound a failed monitor-ci pipeline for this SHA, will retry the failed jobs\n"
|
||||
else
|
||||
printf "\nno passing monitor-ci pipelines found for this SHA, starting a new one\n"
|
||||
fi
|
||||
|
|
@ -87,40 +174,52 @@ if [[ -z "${OSS_SHA:-}" ]]; then
|
|||
# set the parameters for starting the monitor-ci pipeline from the UI repo
|
||||
DEPLOY_PROD=false
|
||||
if [[ "${UI_BRANCH}" == "master" ]]; then
|
||||
DEPLOY_PROD=false # TODO: change this to true when we're ready to depend on this script
|
||||
# In order to deploy to prod:
|
||||
# - UI_BRANCH must be 'master'
|
||||
# - MONITOR_CI_BRANCH must be 'master'
|
||||
# - DEPLOY_PROD must be 'true'
|
||||
# - UI_GITHUB_ORG must be 'influxdata'
|
||||
# UI_BRANCH, DEPLOY_PROD are both neccesary to ensure that UI_BRANCH parameter's default in the monitor-ci pipeline doesn't deploy us to production unless started by this script.
|
||||
DEPLOY_PROD=true
|
||||
fi
|
||||
if [ -n "${PULL_REQUEST}" ]; then
|
||||
# if running from pull request, use github api to get source org and branch.
|
||||
# this is needed to support forks of the UI repo.
|
||||
pr=$(echo ${PULL_REQUEST} | grep -oE "[^/]+$")
|
||||
github=$(curl -s --fail --request GET \
|
||||
--url https://api.github.com/repos/influxdata/ui/pulls/${pr})
|
||||
UI_BRANCH=$(echo ${github} | jq -r '.head.ref')
|
||||
UI_GITHUB_ORG=$(echo ${github} | jq -r '.head.repo.owner.login')
|
||||
fi
|
||||
|
||||
pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH}, UI branch ${UI_BRANCH} and using UI SHA ${SHA}"
|
||||
reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"ui-sha\":\"${SHA}\", \"ui-branch\":\"${UI_BRANCH}\", \"ui-pull-request\":\"${PULL_REQUEST}\", \"deploy-prod\":${DEPLOY_PROD}}}"
|
||||
reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"ui-sha\":\"${SHA}\", \"ui-github-org\":\"${UI_GITHUB_ORG:-influxdata}\", \"ui-branch\":\"${UI_BRANCH}\", \"ui-pull-request\":\"${PULL_REQUEST}\", \"ui-pr-author\":\"${CIRCLE_USERNAME}\", \"deploy-prod\":${DEPLOY_PROD}}}"
|
||||
else
|
||||
# set the parameters for starting the monitor-ci pipeline from the influxdb repo
|
||||
pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH} using OSS SHA ${OSS_SHA}"
|
||||
reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"oss-sha\":\"${OSS_SHA}\" }}"
|
||||
# if OSS_SHA parameter is set, run a minimal pipeline for OSS-specific testing.
|
||||
if [[ -z "${SHA:-}" ]]; then
|
||||
# no UI sha means to run the tests through the latest UI docker image with the OSS API backend based on the provided OSS commit SHA
|
||||
required_workflows=( "build-oss" )
|
||||
pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH} using OSS SHA ${OSS_SHA}"
|
||||
reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"oss-sha\":\"${OSS_SHA}\" }}"
|
||||
else
|
||||
# if both a UI and OSS SHA are provided, run with the OSS backend with an embedded UI, and run the tests on it built from the commit referenced by the UI SHA..
|
||||
required_workflows=( "build-oss-embedded" )
|
||||
pipelineStartMsg="starting monitor-ci pipeline targeting monitor-ci branch ${MONITOR_CI_BRANCH} using OSS SHA ${OSS_SHA} and UI SHA ${SHA}"
|
||||
reqData="{\"branch\":\"${MONITOR_CI_BRANCH}\", \"parameters\":{ \"ui-sha\":\"${SHA}\", \"oss-sha\":\"${OSS_SHA}\", \"ui-branch\":\"${UI_BRANCH}\" }}"
|
||||
fi
|
||||
fi
|
||||
|
||||
printf "\n${pipelineStartMsg}\n"
|
||||
pipeline=$(curl -s --fail --request POST \
|
||||
--url https://circleci.com/api/v2/project/gh/influxdata/monitor-ci/pipeline \
|
||||
--header "Circle-Token: ${API_KEY}" \
|
||||
--header 'content-type: application/json' \
|
||||
--header 'Accept: application/json' \
|
||||
--data "${reqData}")
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "failed to start the monitor-ci pipeline, quitting"
|
||||
exit 1
|
||||
# start a new pipeline if we didn't find an existing one to retry
|
||||
if [ $found_failed_pipeline -eq 0 ]; then
|
||||
startNewPipeline "${pipelineStartMsg}" "${reqData}"
|
||||
else
|
||||
retryFailedPipeline ${failed_pipeline_workflow_id} ${failed_pipeline_id} ${failed_pipeline_number}
|
||||
fi
|
||||
|
||||
pipeline_id=$(echo ${pipeline} | jq -r '.id')
|
||||
pipeline_number=$(echo ${pipeline} | jq -r '.number')
|
||||
|
||||
printf "\nwaiting for monitor-ci pipeline to begin...\n"
|
||||
sleep 1m
|
||||
printf "\nmonitor-ci pipeline has begun. Running pipeline number ${pipeline_number} with id ${pipeline_id}\n"
|
||||
|
||||
# poll the status of the monitor-ci pipeline
|
||||
is_failure=0
|
||||
attempts=0
|
||||
max_attempts=40 # minutes
|
||||
max_attempts=30 # minutes
|
||||
while [ $attempts -le $max_attempts ];
|
||||
do
|
||||
|
||||
|
|
@ -134,18 +233,19 @@ do
|
|||
|
||||
# when the pipeline has finished
|
||||
if [ ${number_running_workflows} -eq 0 ]; then
|
||||
workflows_ids=( $(echo ${workflows} | jq -r '.items | .[].id') )
|
||||
# report failed jobs per required workflow
|
||||
for required_workflow_name in "${required_workflows[@]}"; do
|
||||
workflow_id=$(echo ${workflows} | jq -r --arg name "${required_workflow_name}" '.items | map(select(.name == $name and .status == "success")) | .[].id')
|
||||
|
||||
# report failed jobs per workflow
|
||||
for workflow_id in "${workflows_ids[@]}"; do
|
||||
workflow_status=$(echo ${workflows} | jq -r --arg id "${workflow_id}" '.items | map(select(.id == $id)) | .[].status')
|
||||
|
||||
if [[ "$workflow_status" == "success" ]]; then
|
||||
if [ -n "${workflow_id}" ]; then
|
||||
printf "\nSUCCESS: monitor-ci workflow with id ${workflow_id} passed: https://app.circleci.com/pipelines/github/influxdata/monitor-ci/${pipeline_number}/workflows/${workflow_id} \n"
|
||||
else
|
||||
# set job failure
|
||||
is_failure=1
|
||||
|
||||
# get the workflow_id of this failed required workflow (if there are multiple, get the most recent one)
|
||||
workflow_id=$(echo ${workflows} | jq -r --arg name "${required_workflow_name}" '.items |= sort_by(.created_at) | .items | map(select(.name == $name and .status == "failed")) | .[-1].id')
|
||||
|
||||
# get the jobs that failed for this workflow
|
||||
jobs=$(curl -s --request GET \
|
||||
--url "https://circleci.com/api/v2/workflow/${workflow_id}/job" \
|
||||
|
|
@ -194,7 +294,13 @@ do
|
|||
subdirectory="oss"
|
||||
fi
|
||||
fi
|
||||
output="monitor-ci/test-artifacts/results/${name}/${subdirectory}/${filename}"
|
||||
safeName="${name//\//-}"
|
||||
if [[ "${safeName}" == *"remocal"* ]]; then
|
||||
# put all remocal artifacts in the same parent directory
|
||||
safeName="remocal/${safeName}"
|
||||
fi
|
||||
mkdir -p "monitor-ci/test-artifacts/results/${safeName}/${subdirectory}"
|
||||
output="monitor-ci/test-artifacts/results/${safeName}/${subdirectory}/${filename}"
|
||||
curl -L -s --request GET \
|
||||
--output "${output}" \
|
||||
--url "${url}" \
|
||||
|
|
@ -204,8 +310,10 @@ do
|
|||
fi
|
||||
done
|
||||
|
||||
printf "\n\nFAILURE: monitor-ci workflow with id ${workflow_id} failed:\n"
|
||||
printf "https://app.circleci.com/pipelines/github/influxdata/monitor-ci/${pipeline_number}/workflows/${workflow_id}"
|
||||
printf "\n\nFAILURE: monitor-ci workflow with id ${workflow_id} failed.\n"
|
||||
printf "\n********************************************************\n"
|
||||
printf "monitor-ci pipeline link: \nhttps://app.circleci.com/pipelines/github/influxdata/monitor-ci/${pipeline_number}/workflows/${workflow_id}\n"
|
||||
printf "\n********************************************************\n"
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue