diff --git a/hack/test-flake-chart/collect_data.sh b/hack/test-flake-chart/collect_data.sh index 392aeb7c9d..644c7842de 100755 --- a/hack/test-flake-chart/collect_data.sh +++ b/hack/test-flake-chart/collect_data.sh @@ -1,28 +1,8 @@ #!/bin/bash -# Create temp path for partial data (storing everything but the commit date.) -PARTIAL_DATA_PATH=$(mktemp) -# Write -echo "Partial path: $PARTIAL_DATA_PATH" 1>&2 - -# Print header. -printf "Commit Hash,Commit Date,Environment,Test,Status,Duration\n" +DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) # 1) "cat" together all summary files. -# 2) Turn each test in each summary file to a CSV line containing its commit hash, environment, test, and status. -# 3) Copy partial data to $PARTIAL_DATA_PATH to join with date later. -# 4) Extract only commit hash for each row -# 5) Make the commit hashes unique (we assume that gsutil cats files from the same hash next to each other). -# Also force buffering to occur per line so remainder of pipe can continue to process. -# 6) Execute git log for each commit to get the date of each. -# 7) Join dates with test data. +# 2) Process all summary files. gsutil cat gs://minikube-builds/logs/master/*/*_summary.json \ -| jq -r '((.PassedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: .Durations[$name], status: "Passed"}), - (.FailedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: .Durations[$name], status: "Failed"}), - (.SkippedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: 0, status: "Skipped"})) - | .commit + "," + .environment + "," + .test + "," + .status + "," + (.duration | tostring)' \ -| tee $PARTIAL_DATA_PATH \ -| sed -r -n 's/^([^,]+),.*/\1/p' \ -| stdbuf -oL -eL uniq \ -| xargs -I {} git log -1 --pretty=format:"{},%as%n" {} \ -| join -t "," - $PARTIAL_DATA_PATH +| $DIR/process_data.sh diff --git a/hack/test-flake-chart/process_data.sh b/hack/test-flake-chart/process_data.sh new file mode 100755 index 0000000000..7348c1b178 --- /dev/null +++ b/hack/test-flake-chart/process_data.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Create temp path for partial data (storing everything but the commit date.) +PARTIAL_DATA_PATH=$(mktemp) +# Print the partial path for debugging/convenience. +echo "Partial path: $PARTIAL_DATA_PATH" 1>&2 + +# Print header. +printf "Commit Hash,Commit Date,Environment,Test,Status,Duration\n" + +# 1) Turn each test in each summary file to a CSV line containing its commit hash, environment, test, and status. +# 2) Copy partial data to $PARTIAL_DATA_PATH to join with date later. +# 3) Extract only commit hash for each row +# 4) Make the commit hashes unique (we assume that gsutil cats files from the same hash next to each other). +# Also force buffering to occur per line so remainder of pipe can continue to process. +# 5) Execute git log for each commit to get the date of each. +# 6) Join dates with test data. +jq -r '((.PassedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: .Durations[$name], status: "Passed"}), + (.FailedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: .Durations[$name], status: "Failed"}), + (.SkippedTests[]? as $name | {commit: .Detail.Details, environment: .Detail.Name, test: $name, duration: 0, status: "Skipped"})) + | .commit + "," + .environment + "," + .test + "," + .status + "," + (.duration | tostring)' \ +| tee $PARTIAL_DATA_PATH \ +| sed -r -n 's/^([^,]+),.*/\1/p' \ +| stdbuf -oL -eL uniq \ +| xargs -I {} git log -1 --pretty=format:"{},%as%n" {} \ +| join -t "," - $PARTIAL_DATA_PATH