Merge pull request #12065 from andriyDev/UseJobId

Use job id for logs instead of short commit hash
pull/12069/head
Medya Ghazizadeh 2021-07-28 13:30:50 -07:00 committed by GitHub
commit d69a98c45c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 43 additions and 54 deletions

View File

@ -28,8 +28,8 @@ This proposal is for a system to inform users, both public and internal, of the
### Overview
The full overview of the system is as follows:
* The `minikube` Jenkins job builds all binaries for integration tests. On completion, it triggers `minikube_set_pending.sh`, which updates the PR status of integration tests to pending. In addition, `minikube_set_pending.sh` will upload the list of environments to wait for to `gs://minikube-builds/logs/<MINIKUBE_LOCATION>/<COMMIT>/started_environments_<minikube_BUILD_NUMBER>.txt`
* Jenkins integration test jobs running on master generate gopogh summaries. Each job then triggers `Flake Rate Upload` which appends the completed environment to `gs://minikube-builds/logs/<MINIKUBE_LOCATION>/<COMMIT>/finished_environments_<minikube_BUILD_NUMBER>.txt`
* The `minikube` Jenkins job builds all binaries for integration tests. On completion, it triggers `minikube_set_pending.sh`, which updates the PR status of integration tests to pending. In addition, `minikube_set_pending.sh` will upload the list of environments to wait for to `gs://minikube-builds/logs/<MINIKUBE_LOCATION>/<minikube_BUILD_NUMBER>/started_environments.txt`
* Jenkins integration test jobs running on master generate gopogh summaries. Each job then triggers `Flake Rate Upload` which appends the completed environment to `gs://minikube-builds/logs/<MINIKUBE_LOCATION>/<minikube_BUILD_NUMBER>/finished_environments.txt`
* Once all started environments are present in finished environments, if running on master, all gopogh reports are processed through `upload_tests.sh` and appended into the dataset of all test runs at `gs://minikube-flake-rate/data.csv`. If running on a PR, the gopogh reports are used with `report_flakes.sh` to write a comment on PRs about the flake rates of all failed tests.
* A Jenkins job runs regularly to compute the flake rates of tests in `gs://minikube-flake-rate/data.csv` and outputs the results into `gs://minikube-flake-rate/flake_rates.csv`, including the environment (e.g. `Docker_Linux`), the test name, the flake rate as a percentage, and the average duration
* An HTML+JS file, hosted on `gs://minikube-flake-rate/flake_chart.html`, will read the full test data (`gs://minikube-flake-rate/data.csv`), and parse it into a chart displaying the daily flake rates and average durations of the requested tests (specified by url query arguments)

View File

@ -48,7 +48,7 @@ if [ $ec -gt 0 ]; then
gh pr comment ${ghprbPullId} --body "Hi ${ghprbPullAuthorLoginMention}, building a new ISO failed.
See the logs at:
```
https://storage.cloud.google.com/minikube-builds/logs/${ghprbPullId}/${ghprbActualCommit:0:7}/iso_build.txt
https://storage.cloud.google.com/minikube-builds/logs/${ghprbPullId}/iso-${BUILD_NUMBER}/iso_build.txt
```
"
fi

View File

@ -63,7 +63,7 @@ If($env:status -eq "failure") {
echo $description
$env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT"
$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:ROOT_JOB_ID"
#Upload logs to gcs
gsutil -qm cp testout.txt gs://$gcs_bucket/${env:JOB_NAME}out.txt

View File

@ -36,7 +36,7 @@ export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
readonly TIMEOUT=${1:-120m}
public_log_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${COMMIT:0:7}/${JOB_NAME}.html"
public_log_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${ROOT_JOB_ID}/${JOB_NAME}.html"
# retry_github_status provides reliable github status updates
function retry_github_status() {
@ -434,7 +434,7 @@ REPORT_URL_BASE="https://storage.googleapis.com"
if [ -z "${EXTERNAL}" ]; then
# If we're already in GCP, then upload results to GCS directly
SHORT_COMMIT=${COMMIT:0:7}
JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${SHORT_COMMIT}/${JOB_NAME}"
JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${ROOT_JOB_ID}/${JOB_NAME}"
echo ">> Copying ${TEST_OUT} to gs://${JOB_GCS_BUCKET}.out.txt"
echo ">> public URL: ${REPORT_URL_BASE}/${JOB_GCS_BUCKET}.out.txt"

View File

@ -70,7 +70,7 @@ if [ $ec -gt 0 ]; then
gh pr comment ${ghprbPullId} --body "Hi ${ghprbPullAuthorLoginMention}, building a new kicbase image failed.
See the logs at:
```
https://storage.cloud.google.com/minikube-builds/logs/${ghprbPullId}/${ghprbActualCommit:0:7}/kic_image_build.txt
https://storage.cloud.google.com/minikube-builds/logs/${ghprbPullId}/kicbase-${BUILD_NUMBER}/kic_image_build.txt
```
"
fi

View File

@ -49,8 +49,7 @@ jobs=(
'Docker_Cloud_Shell'
)
SHORT_COMMIT=${THE_COMMIT:0:7}
STARTED_LIST_REMOTE="gs://minikube-builds/logs/${ghprbPullId}/${SHORT_COMMIT}/started_environments_${BUILD_NUMBER}.txt"
STARTED_LIST_REMOTE="gs://minikube-builds/logs/${ghprbPullId}/${BUILD_NUMBER}/started_environments.txt"
printf "%s\n" "${jobs[@]}" | gsutil cp - "${STARTED_LIST_REMOTE}"
if [ "${ghprbPullId}" == "master" ]; then
@ -94,5 +93,5 @@ function retry_github_status() {
for j in ${jobs[@]}; do
retry_github_status "${ghprbActualCommit}" "${j}" "pending" "${access_token}" \
"https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${SHORT_COMMIT}/${j}.pending"
"https://storage.googleapis.com/minikube-builds/logs/${ghprbPullId}/${BUILD_NUMBER}/${j}.pending"
done

View File

@ -195,15 +195,10 @@ function aggregateRuns(testRuns) {
date: tests[0].date, // Get one of the dates from the tests (which will all be the same).
flakeRate: tests.map(test => test.status === testStatus.FAILED ? 100 : 0).average(), // Compute average of runs where FAILED counts as 100%.
duration: tests.map(test => test.duration).average(), // Compute average duration of runs.
commitHashes: tests.map(test => ({ // Take all hashes, statuses, and durations of tests in this group.
hash: test.commit,
jobs: tests.map(test => ({ // Take all job ids, statuses, and durations of tests in this group.
id: test.rootJob,
status: test.status,
duration: test.duration
})).groupBy(run => run.hash).map(runsWithSameHash => ({
hash: runsWithSameHash[0].hash,
failures: runsWithSameHash.map(run => run.status === testStatus.FAILED ? 1 : 0).sum(),
runs: runsWithSameHash.length,
duration: runsWithSameHash.map(run => run.duration).average(),
}))
}));
}
@ -220,20 +215,15 @@ function aggregateWeeklyRuns(testRuns, weekDates) {
date: weekDates.findRounded(tests[0].date), // Get one of the dates from the tests, and use it to get the rounded time (which will all be the same).
flakeRate: tests.map(test => test.status === testStatus.FAILED ? 100 : 0).average(), // Compute average of runs where FAILED counts as 100%.
duration: tests.map(test => test.duration).average(), // Compute average duration of runs.
commitHashes: tests.map(test => ({ // Take all hashes, statuses, and durations of tests in this group.
hash: test.commit,
jobs: tests.map(test => ({ // Take all job ids, statuses, and durations of tests in this group.
id: test.rootJob,
status: test.status,
duration: test.duration
})).groupBy(run => run.hash).map(runsWithSameHash => ({
hash: runsWithSameHash[0].hash,
failures: runsWithSameHash.map(run => run.status === testStatus.FAILED ? 1 : 0).sum(),
runs: runsWithSameHash.length,
duration: runsWithSameHash.map(run => run.duration).average(),
}))
}));
}
const hashToLink = (hash, environment) => `https://storage.googleapis.com/minikube-builds/logs/master/${hash.substring(0,7)}/${environment}.html`;
const jobIdToLink = (jobId, environment) => `https://storage.googleapis.com/minikube-builds/logs/master/${jobId}/${environment}.html`;
function displayTestAndEnvironmentChart(testData, testName, environmentName) {
const testRuns = testData
@ -256,15 +246,15 @@ function displayTestAndEnvironmentChart(testData, testName, environmentName) {
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${groupData.date.toString()}</b><br>
<b>Flake Percentage:</b> ${groupData.flakeRate.toFixed(2)}%<br>
<b>Hashes:</b><br>
${groupData.commitHashes.map(({ hash, failures, runs }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Failures: ${failures}/${runs})`).join("<br>")}
<b>Jobs:</b><br>
${groupData.jobs.map(({ id, status }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${status})`).join("<br>")}
</div>`,
groupData.duration,
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${groupData.date.toString()}</b><br>
<b>Average Duration:</b> ${groupData.duration.toFixed(2)}s<br>
<b>Hashes:</b><br>
${groupData.commitHashes.map(({ hash, runs, duration }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Average of ${runs}: ${duration.toFixed(2)}s)`).join("<br>")}
<b>Jobs:</b><br>
${groupData.jobs.map(({ id, duration }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${duration}s)`).join("<br>")}
</div>`,
])
);
@ -332,15 +322,15 @@ function displayTestAndEnvironmentChart(testData, testName, environmentName) {
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${groupData.date.toString()}</b><br>
<b>Flake Percentage:</b> ${groupData.flakeRate.toFixed(2)}%<br>
<b>Hashes:</b><br>
${groupData.commitHashes.map(({ hash, failures, runs }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Failures: ${failures}/${runs})`).join("<br>")}
<b>Jobs:</b><br>
${groupData.jobs.map(({ id, status }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${status})`).join("<br>")}
</div>`,
groupData.duration,
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${groupData.date.toString()}</b><br>
<b>Average Duration:</b> ${groupData.duration.toFixed(2)}s<br>
<b>Hashes:</b><br>
${groupData.commitHashes.map(({ hash, runs, duration }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Average of ${runs}: ${duration.toFixed(2)}s)`).join("<br>")}
<b>Jobs:</b><br>
${groupData.jobs.map(({ id, duration }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${duration}s)`).join("<br>")}
</div>`,
])
);
@ -434,7 +424,7 @@ function displayEnvironmentChart(testData, environmentName) {
const dateInfo = data.get(date);
return dateInfo === undefined ? null : {
flakeRate: dateInfo.flakeRate,
runs: dateInfo.commitHashes.length
runs: dateInfo.jobs.length
};
}).filter(dateInfo => dateInfo != null)
.reduce(({flakeCount, totalCount}, {flakeRate, runs}) => ({
@ -474,8 +464,8 @@ function displayEnvironmentChart(testData, environmentName) {
<b style="display: block">${name}</b><br>
<b>${data.date.toString()}</b><br>
<b>Flake Percentage:</b> ${data.flakeRate.toFixed(2)}%<br>
<b>Hashes:</b><br>
${data.commitHashes.map(({ hash, failures, runs }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Failures: ${failures}/${runs})`).join("<br>")}
<b>Jobs:</b><br>
${data.jobs.map(({ id, status }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${status})`).join("<br>")}
</div>`
] : [null, null];
})).flat())
@ -551,8 +541,8 @@ function displayEnvironmentChart(testData, environmentName) {
<b style="display: block">${name}</b><br>
<b>${data.date.toString()}</b><br>
<b>Flake Percentage:</b> ${data.flakeRate.toFixed(2)}%<br>
<b>Hashes:</b><br>
${data.commitHashes.map(({ hash, failures, runs }) => ` - <a href="${hashToLink(hash, environmentName)}">${hash}</a> (Failures: ${failures}/${runs})`).join("<br>")}
<b>Jobs:</b><br>
${data.jobs.map(({ id, status }) => ` - <a href="${jobIdToLink(id, environmentName)}">${id}</a> (${status})`).join("<br>")}
</div>`
] : [null, null];
})).flat())
@ -611,15 +601,15 @@ function displayEnvironmentChart(testData, environmentName) {
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${dateInfo.date.toString()}</b><br>
<b>Test Count (averaged): </b> ${+dateInfo.testCount.toFixed(2)}<br>
<b>Hashes:</b><br>
${dateInfo.runInfo.map(job => ` - <a href="${hashToLink(job.commit, environmentName)}">${job.commit}</a> (Job ${job.rootJob}) Test count: ${job.testCount}`).join("<br>")}
<b>Jobs:</b><br>
${dateInfo.runInfo.map(job => ` - <a href="${jobIdToLink(job.rootJob, environmentName)}">${job.rootJob}</a> Test count: ${job.testCount}`).join("<br>")}
</div>`,
dateInfo.totalDuration,
`<div style="padding: 1rem; font-family: 'Arial'; font-size: 14">
<b>${dateInfo.date.toString()}</b><br>
<b>Total Duration (averaged): </b> ${+dateInfo.totalDuration.toFixed(2)}<br>
<b>Hashes:</b><br>
${dateInfo.runInfo.map(job => ` - <a href="${hashToLink(job.commit, environmentName)}">${job.commit}</a> (Job ${job.rootJob}) Total Duration: ${+job.totalDuration.toFixed(2)}s`).join("<br>")}
<b>Jobs:</b><br>
${dateInfo.runInfo.map(job => ` - <a href="${jobIdToLink(job.rootJob, environmentName)}">${job.rootJob}</a> Total Duration: ${+job.totalDuration.toFixed(2)}s`).join("<br>")}
</div>`,
]));
const options = {

View File

@ -21,12 +21,12 @@
set -eu -o pipefail
if [ "$#" -ne 3 ]; then
echo "Wrong number of arguments. Usage: report_flakes.sh <PR number> <short commit> <environment list file>" 1>&2
echo "Wrong number of arguments. Usage: report_flakes.sh <PR number> <Root job id> <environment list file>" 1>&2
exit 1
fi
PR_NUMBER=$1
SHORT_COMMIT=$2
ROOT_JOB=$2
ENVIRONMENT_LIST=$3
# To prevent having a super-long comment, add a maximum number of tests to report.
@ -42,7 +42,7 @@ TMP_DATA=$(mktemp)
# 5) Filter tests to only include failed tests (and only get their names and environment).
# 6) Sort by environment, then test name.
# 7) Store in file $TMP_DATA.
< "${ENVIRONMENT_LIST}" sed -r "s|^|gs://minikube-builds/logs/${PR_NUMBER}/${SHORT_COMMIT}/|; s|$|_summary.json|" \
< "${ENVIRONMENT_LIST}" sed -r "s|^|gs://minikube-builds/logs/${PR_NUMBER}/${ROOT_JOB}/|; s|$|_summary.json|" \
| (xargs gsutil ls || true) \
| xargs gsutil cat \
| "$DIR/process_data.sh" \
@ -79,7 +79,7 @@ printf "These are the flake rates of all failed tests.\n|Environment|Failed Test
# Create variables to use for sed command.
ENV_CHART_LINK_FORMAT="https://storage.googleapis.com/minikube-flake-rate/flake_chart.html?env=\1"
TEST_CHART_LINK_FORMAT="${ENV_CHART_LINK_FORMAT}\&test=\2"
TEST_GOPOGH_LINK_FORMAT="https://storage.googleapis.com/minikube-builds/logs/${PR_NUMBER}/${SHORT_COMMIT}/\1.html#fail_\2"
TEST_GOPOGH_LINK_FORMAT="https://storage.googleapis.com/minikube-builds/logs/${PR_NUMBER}/${ROOT_JOB}/\1.html#fail_\2"
# 1) Get the first $MAX_REPORTED_TESTS lines.
# 2) Print a row in the table with the environment, test name, flake rate, and a link to the flake chart for that test.
# 3) Append these rows to file $TMP_COMMENT.

View File

@ -26,8 +26,8 @@
set -o pipefail
BUCKET_PATH="gs://minikube-builds/logs/${MINIKUBE_LOCATION}/${COMMIT:0:7}"
STARTED_LIST=$(gsutil cat "${BUCKET_PATH}/started_environments_${ROOT_JOB_ID}.txt" | sort | uniq)
BUCKET_PATH="gs://minikube-builds/logs/${MINIKUBE_LOCATION}/${ROOT_JOB_ID}"
STARTED_LIST=$(gsutil cat "${BUCKET_PATH}/started_environments.txt" | sort | uniq)
if [ $? -ne 0 ]; then
echo "Unable to read environment list. Likely being run before all tests are ready or after tests have already been uploaded." 1>&2
@ -36,7 +36,7 @@ fi
set -eu -o pipefail
FINISHED_LIST_REMOTE="${BUCKET_PATH}/finished_environments_${ROOT_JOB_ID}.txt"
FINISHED_LIST_REMOTE="${BUCKET_PATH}/finished_environments.txt"
# Ensure FINISHED_LIST_REMOTE exists so we can append (but don't erase any existing entries in FINISHED_LIST_REMOTE)
< /dev/null gsutil cp -n - "${FINISHED_LIST_REMOTE}"
# Copy the job name to APPEND_TMP. If the job name ends in "_integration" remove it.
@ -67,7 +67,7 @@ if [ ${STARTED_COUNT} -ne ${FINISHED_COUNT} ]; then
fi
# Prevent other invocations of this script from uploading the same thing multiple times.
gsutil rm "${BUCKET_PATH}/started_environments_${ROOT_JOB_ID}.txt"
gsutil rm "${BUCKET_PATH}/started_environments.txt"
# At this point, we know all integration tests are done and we can process all summaries safely.
echo "${FINISHED_LIST_JOINED}" > ${FINISHED_LIST}
@ -84,8 +84,8 @@ if [[ "${MINIKUBE_LOCATION}" == "master" ]]; then
"${DIR}/upload_tests.sh" "${SUMMARY}" || true
done
else
"${DIR}/report_flakes.sh" "${MINIKUBE_LOCATION}" "${COMMIT:0:7}" "${FINISHED_LIST}"
"${DIR}/report_flakes.sh" "${MINIKUBE_LOCATION}" "${ROOT_JOB_ID}" "${FINISHED_LIST}"
fi
gsutil rm "${BUCKET_PATH}/finished_environments_${ROOT_JOB_ID}.txt"
gsutil rm "${BUCKET_PATH}/finished_environments.txt"
rm "${FINISHED_LIST}"

View File

@ -26,7 +26,7 @@ set -x
# upload results to GCS
UPSTREAM_JOB=${UPSTREAM_JOB%"_integration"}
JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${COMMIT:0:7}/${UPSTREAM_JOB}"
JOB_GCS_BUCKET="minikube-builds/logs/${MINIKUBE_LOCATION}/${ROOT_JOB_ID}/${UPSTREAM_JOB}"
ARTIFACTS=artifacts/test_reports

View File

@ -21,7 +21,7 @@ gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/setup_docker_des
gsutil.cmd -m cp -r gs://minikube-builds/$env:MINIKUBE_LOCATION/common.ps1 out/
$env:SHORT_COMMIT=$env:COMMIT.substring(0, 7)
$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:SHORT_COMMIT"
$gcs_bucket="minikube-builds/logs/$env:MINIKUBE_LOCATION/$env:ROOT_JOB_ID"
./out/setup_docker_desktop_windows.ps1
If ($lastexitcode -gt 0) {