diff --git a/hack/jenkins/test-flake-chart/compute_flake_rate.go b/hack/jenkins/test-flake-chart/compute_flake_rate.go index ccecf4f810..0025df2fbe 100644 --- a/hack/jenkins/test-flake-chart/compute_flake_rate.go +++ b/hack/jenkins/test-flake-chart/compute_flake_rate.go @@ -24,6 +24,7 @@ import ( "os" "runtime/debug" "sort" + "strconv" "strings" "time" ) @@ -45,10 +46,12 @@ func main() { splitEntries := splitData(testEntries) filteredEntries := filterRecentEntries(splitEntries, *dateRange) flakeRates := computeFlakeRates(filteredEntries) - fmt.Println("Environment,Test,Flake Rate") + averageDurations := computeAverageDurations(filteredEntries) + fmt.Println("Environment,Test,Flake Rate,Duration") for environment, environmentSplit := range flakeRates { for test, flakeRate := range environmentSplit { - fmt.Printf("%s,%s,%.2f\n", environment, test, flakeRate*100) + duration := averageDurations[environment][test] + fmt.Printf("%s,%s,%.2f,%.3f\n", environment, test, flakeRate*100, duration) } } } @@ -59,12 +62,14 @@ func main() { // environment: "Docker_Linux", // date: time.Now, // status: "Passed", +// duration: 0.1, // } type testEntry struct { name string environment string date time.Time status string + duration float32 } // A map with keys of (environment, test_name) to values of slcies of TestEntry. @@ -107,12 +112,19 @@ func readData(file io.Reader) []testEntry { date, err := time.Parse("2006-01-02", fields[1]) if err != nil { fmt.Printf("Failed to parse date: %v\n", err) + continue + } + duration, err := strconv.ParseFloat(fields[5], 32) + if err != nil { + fmt.Printf("Failed to parse duration: %v\n", err) + continue } testEntries = append(testEntries, testEntry{ name: fields[3], environment: fields[2], date: date, status: fields[4], + duration: float32(duration), }) } } @@ -215,14 +227,32 @@ func computeFlakeRates(splitEntries splitEntryMap) map[string]map[string]float32 return flakeRates } -// Sets the `value` of keys `environment` and `test` in `flakeRates`. -func setValue(flakeRates map[string]map[string]float32, environment, test string, value float32) { +// Computes the average durations over each entry in `splitEntries`. +func computeAverageDurations(splitEntries splitEntryMap) map[string]map[string]float32 { + averageDurations := make(map[string]map[string]float32) + for environment, environmentSplit := range splitEntries { + for test, testSplit := range environmentSplit { + durationSum := float32(0) + for _, entry := range testSplit { + durationSum += entry.duration + } + if len(testSplit) != 0 { + durationSum /= float32(len(testSplit)) + } + setValue(averageDurations, environment, test, durationSum) + } + } + return averageDurations +} + +// Sets the `value` of keys `environment` and `test` in `mapEntries`. +func setValue(mapEntries map[string]map[string]float32, environment, test string, value float32) { // Lookup the environment. - environmentRates, ok := flakeRates[environment] + environmentRates, ok := mapEntries[environment] if !ok { // If the environment map is missing, make a map for this environment and store it. environmentRates = make(map[string]float32) - flakeRates[environment] = environmentRates + mapEntries[environment] = environmentRates } environmentRates[test] = value } diff --git a/hack/jenkins/test-flake-chart/compute_flake_rate_test.go b/hack/jenkins/test-flake-chart/compute_flake_rate_test.go index 2f458daad9..d4013c0885 100644 --- a/hack/jenkins/test-flake-chart/compute_flake_rate_test.go +++ b/hack/jenkins/test-flake-chart/compute_flake_rate_test.go @@ -53,10 +53,10 @@ func TestReadData(t *testing.T) { actualData := readData(strings.NewReader( `A,B,C,D,E,F hash,2000-01-01,env1,test1,Passed,1 - hash,2001-01-01,env2,test2,Failed,1 - hash,,,test1,,1 - hash,2002-01-01,,,Passed,1 - hash,2003-01-01,env3,test3,Passed,1`, + hash,2001-01-01,env2,test2,Failed,0.5 + hash,,,test1,,0.6 + hash,2002-01-01,,,Passed,0.9 + hash,2003-01-01,env3,test3,Passed,2`, )) expectedData := []testEntry{ { @@ -64,30 +64,35 @@ func TestReadData(t *testing.T) { environment: "env1", date: simpleDate(2000, 1), status: "Passed", + duration: 1, }, { name: "test2", environment: "env2", date: simpleDate(2001, 1), status: "Failed", + duration: 0.5, }, { name: "test1", environment: "env2", date: simpleDate(2001, 1), status: "Failed", + duration: 0.6, }, { name: "test1", environment: "env2", date: simpleDate(2002, 1), status: "Passed", + duration: 0.9, }, { name: "test3", environment: "env3", date: simpleDate(2003, 1), status: "Passed", + duration: 2, }, } @@ -280,6 +285,42 @@ func TestFilterRecentEntries(t *testing.T) { compareSplitData(t, actualData, expectedData) } +func compareValues(t *testing.T, actualValues, expectedValues map[string]map[string]float32) { + for environment, actualTests := range actualValues { + expectedTests, environmentOk := expectedValues[environment] + if !environmentOk { + t.Errorf("Unexpected environment %s in actual", environment) + continue + } + + for test, actualValue := range actualTests { + expectedValue, testOk := expectedTests[test] + if !testOk { + t.Errorf("Unexpected test %s (in environment %s) in actual", test, environment) + continue + } + + if actualValue != expectedValue { + t.Errorf("Wrong value at environment %s and test %s. Expected: %v, Actual: %v", environment, test, expectedValue, actualValue) + } + } + + for test := range expectedTests { + _, testOk := actualTests[test] + if !testOk { + t.Errorf("Missing expected test %s (in environment %s) in actual", test, environment) + } + } + } + + for environment := range expectedValues { + _, environmentOk := actualValues[environment] + if !environmentOk { + t.Errorf("Missing expected environment %s in actual", environment) + } + } +} + func TestComputeFlakeRates(t *testing.T) { actualData := computeFlakeRates(splitEntryMap{ "env1": { @@ -357,37 +398,95 @@ func TestComputeFlakeRates(t *testing.T) { }, } - for environment, actualTests := range actualData { - expectedTests, environmentOk := expectedData[environment] - if !environmentOk { - t.Errorf("Unexpected environment %s in actual", environment) - continue - } - - for test, actualFlakeRate := range actualTests { - expectedFlakeRate, testOk := expectedTests[test] - if !testOk { - t.Errorf("Unexpected test %s (in environment %s) in actual", test, environment) - continue - } - - if actualFlakeRate != expectedFlakeRate { - t.Errorf("Wrong flake rate. Expected: %v, Actual: %v", expectedFlakeRate, actualFlakeRate) - } - } - - for test := range expectedTests { - _, testOk := actualTests[test] - if !testOk { - t.Errorf("Missing expected test %s (in environment %s) in actual", test, environment) - } - } - } - - for environment := range expectedData { - _, environmentOk := actualData[environment] - if !environmentOk { - t.Errorf("Missing expected environment %s in actual", environment) - } - } + compareValues(t, actualData, expectedData) +} + +func TestComputeAverageDurations(t *testing.T) { + actualData := computeAverageDurations(splitEntryMap{ + "env1": { + "test1": { + { + name: "test1", + environment: "env1", + date: simpleDate(2000, 4), + status: "Passed", + duration: 1, + }, { + name: "test1", + environment: "env1", + date: simpleDate(2000, 3), + status: "Passed", + duration: 2, + }, { + name: "test1", + environment: "env1", + date: simpleDate(2000, 3), + status: "Passed", + duration: 3, + }, { + name: "test1", + environment: "env1", + date: simpleDate(2000, 2), + status: "Passed", + duration: 3, + }, { + name: "test1", + environment: "env1", + date: simpleDate(2000, 1), + status: "Failed", + duration: 3, + }, + }, + "test2": { + { + name: "test2", + environment: "env1", + date: simpleDate(2001, 3), + status: "Failed", + duration: 1, + }, { + name: "test2", + environment: "env1", + date: simpleDate(2001, 2), + status: "Failed", + duration: 3, + }, { + name: "test2", + environment: "env1", + date: simpleDate(2001, 1), + status: "Failed", + duration: 3, + }, + }, + }, + "env2": { + "test2": { + { + name: "test2", + environment: "env2", + date: simpleDate(2003, 3), + status: "Passed", + duration: 0.5, + }, testEntry{ + name: "test2", + environment: "env2", + date: simpleDate(2003, 2), + status: "Failed", + duration: 1.5, + }, + }, + }, + }) + + expectedData := map[string]map[string]float32{ + "env1": { + "test1": float32(12) / float32(5), + "test2": float32(7) / float32(3), + }, + "env2": { + "test2": 1, + }, + } + + compareValues(t, actualData, expectedData) } diff --git a/hack/jenkins/test-flake-chart/report_flakes.sh b/hack/jenkins/test-flake-chart/report_flakes.sh index 85589babf8..a04c0d359c 100755 --- a/hack/jenkins/test-flake-chart/report_flakes.sh +++ b/hack/jenkins/test-flake-chart/report_flakes.sh @@ -54,7 +54,7 @@ TMP_FAILED_RATES="$TMP_FLAKE_RATES\_filtered" # 3) Join the flake rates with the failing tests to only get flake rates of failing tests. # 4) Sort failed test flake rates based on the flakiness of that test - stable tests should be first on the list. # 5) Store in file $TMP_FAILED_RATES. -< "$TMP_FLAKE_RATES" sed -n -r -e "s/$ENVIRONMENT,([a-zA-Z\/_-]*),([.0-9]*)/\1,\2/p" \ +< "$TMP_FLAKE_RATES" sed -n -r -e "s/$ENVIRONMENT,([a-zA-Z\/_-]*),([.0-9]*),[.0-9]*/\1,\2/p" \ | sort -t, -k1,1 \ | join -t , -j 1 "$TMP_DATA" - \ | sort -g -t, -k2,2 \