kubernetes
/
minikube
mirror of https://github.com/kubernetes/minikube.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Make compute_flake_rate also compute average test duration.

pull/11602/head
Andriy Dzikh 2021-06-10 14:00:47 -07:00
parent 728229c719
commit 1c1fdbff42
3 changed files with 173 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
hack/jenkins/test-flake-chart/compute_flake_rate.go
View File

@ -24,6 +24,7 @@ import (
"os"
"runtime/debug"
"sort"
"strconv"
"strings"
"time"
)
@ -45,10 +46,12 @@ func main() {
splitEntries := splitData(testEntries)
filteredEntries := filterRecentEntries(splitEntries, *dateRange)
flakeRates := computeFlakeRates(filteredEntries)
fmt.Println("Environment,Test,Flake Rate")
averageDurations := computeAverageDurations(filteredEntries)
fmt.Println("Environment,Test,Flake Rate,Duration")
for environment, environmentSplit := range flakeRates {
for test, flakeRate := range environmentSplit {
fmt.Printf("%s,%s,%.2f\n", environment, test, flakeRate*100)
duration := averageDurations[environment][test]
fmt.Printf("%s,%s,%.2f,%.3f\n", environment, test, flakeRate*100, duration)
}
}
}
@ -59,12 +62,14 @@ func main() {
// environment: "Docker_Linux",
// date: time.Now,
// status: "Passed",
// duration: 0.1,
// }
type testEntry struct {
name string
environment string
date time.Time
status string
duration float32
}
// A map with keys of (environment, test_name) to values of slcies of TestEntry.
@ -107,12 +112,19 @@ func readData(file io.Reader) []testEntry {
date, err := time.Parse("2006-01-02", fields[1])
if err != nil {
fmt.Printf("Failed to parse date: %v\n", err)
continue
}
duration, err := strconv.ParseFloat(fields[5], 32)
if err != nil {
fmt.Printf("Failed to parse duration: %v\n", err)
continue
}
testEntries = append(testEntries, testEntry{
name: fields[3],
environment: fields[2],
date: date,
status: fields[4],
duration: float32(duration),
})
}
}
@ -215,14 +227,32 @@ func computeFlakeRates(splitEntries splitEntryMap) map[string]map[string]float32
return flakeRates
}
// Sets the `value` of keys `environment` and `test` in `flakeRates`.
func setValue(flakeRates map[string]map[string]float32, environment, test string, value float32) {
// Computes the average durations over each entry in `splitEntries`.
func computeAverageDurations(splitEntries splitEntryMap) map[string]map[string]float32 {
averageDurations := make(map[string]map[string]float32)
for environment, environmentSplit := range splitEntries {
for test, testSplit := range environmentSplit {
durationSum := float32(0)
for _, entry := range testSplit {
durationSum += entry.duration
}
if len(testSplit) != 0 {
durationSum /= float32(len(testSplit))
}
setValue(averageDurations, environment, test, durationSum)
}
}
return averageDurations
}
// Sets the `value` of keys `environment` and `test` in `mapEntries`.
func setValue(mapEntries map[string]map[string]float32, environment, test string, value float32) {
// Lookup the environment.
environmentRates, ok := flakeRates[environment]
environmentRates, ok := mapEntries[environment]
if !ok {
// If the environment map is missing, make a map for this environment and store it.
environmentRates = make(map[string]float32)
flakeRates[environment] = environmentRates
mapEntries[environment] = environmentRates
}
environmentRates[test] = value
}

173
hack/jenkins/test-flake-chart/compute_flake_rate_test.go
View File

@ -53,10 +53,10 @@ func TestReadData(t *testing.T) {
actualData := readData(strings.NewReader(
`A,B,C,D,E,F
hash,2000-01-01,env1,test1,Passed,1
hash,2001-01-01,env2,test2,Failed,1
hash,,,test1,,1
hash,2002-01-01,,,Passed,1
hash,2003-01-01,env3,test3,Passed,1`,
hash,2001-01-01,env2,test2,Failed,0.5
hash,,,test1,,0.6
hash,2002-01-01,,,Passed,0.9
hash,2003-01-01,env3,test3,Passed,2`,
))
expectedData := []testEntry{
{
@ -64,30 +64,35 @@ func TestReadData(t *testing.T) {
environment: "env1",
date: simpleDate(2000, 1),
status: "Passed",
duration: 1,
},
{
name: "test2",
environment: "env2",
date: simpleDate(2001, 1),
status: "Failed",
duration: 0.5,
},
{
name: "test1",
environment: "env2",
date: simpleDate(2001, 1),
status: "Failed",
duration: 0.6,
},
{
name: "test1",
environment: "env2",
date: simpleDate(2002, 1),
status: "Passed",
duration: 0.9,
},
{
name: "test3",
environment: "env3",
date: simpleDate(2003, 1),
status: "Passed",
duration: 2,
},
}
@ -280,6 +285,42 @@ func TestFilterRecentEntries(t *testing.T) {
compareSplitData(t, actualData, expectedData)
}
func compareValues(t *testing.T, actualValues, expectedValues map[string]map[string]float32) {
for environment, actualTests := range actualValues {
expectedTests, environmentOk := expectedValues[environment]
if !environmentOk {
t.Errorf("Unexpected environment %s in actual", environment)
continue
}
for test, actualValue := range actualTests {
expectedValue, testOk := expectedTests[test]
if !testOk {
t.Errorf("Unexpected test %s (in environment %s) in actual", test, environment)
continue
}
if actualValue != expectedValue {
t.Errorf("Wrong value at environment %s and test %s. Expected: %v, Actual: %v", environment, test, expectedValue, actualValue)
}
}
for test := range expectedTests {
_, testOk := actualTests[test]
if !testOk {
t.Errorf("Missing expected test %s (in environment %s) in actual", test, environment)
}
}
}
for environment := range expectedValues {
_, environmentOk := actualValues[environment]
if !environmentOk {
t.Errorf("Missing expected environment %s in actual", environment)
}
}
}
func TestComputeFlakeRates(t *testing.T) {
actualData := computeFlakeRates(splitEntryMap{
"env1": {
@ -357,37 +398,95 @@ func TestComputeFlakeRates(t *testing.T) {
},
}
for environment, actualTests := range actualData {
expectedTests, environmentOk := expectedData[environment]
if !environmentOk {
t.Errorf("Unexpected environment %s in actual", environment)
continue
}
for test, actualFlakeRate := range actualTests {
expectedFlakeRate, testOk := expectedTests[test]
if !testOk {
t.Errorf("Unexpected test %s (in environment %s) in actual", test, environment)
continue
}
if actualFlakeRate != expectedFlakeRate {
t.Errorf("Wrong flake rate. Expected: %v, Actual: %v", expectedFlakeRate, actualFlakeRate)
}
}
for test := range expectedTests {
_, testOk := actualTests[test]
if !testOk {
t.Errorf("Missing expected test %s (in environment %s) in actual", test, environment)
}
}
}
for environment := range expectedData {
_, environmentOk := actualData[environment]
if !environmentOk {
t.Errorf("Missing expected environment %s in actual", environment)
}
}
compareValues(t, actualData, expectedData)
}
func TestComputeAverageDurations(t *testing.T) {
actualData := computeAverageDurations(splitEntryMap{
"env1": {
"test1": {
{
name: "test1",
environment: "env1",
date: simpleDate(2000, 4),
status: "Passed",
duration: 1,
}, {
name: "test1",
environment: "env1",
date: simpleDate(2000, 3),
status: "Passed",
duration: 2,
}, {
name: "test1",
environment: "env1",
date: simpleDate(2000, 3),
status: "Passed",
duration: 3,
}, {
name: "test1",
environment: "env1",
date: simpleDate(2000, 2),
status: "Passed",
duration: 3,
}, {
name: "test1",
environment: "env1",
date: simpleDate(2000, 1),
status: "Failed",
duration: 3,
},
},
"test2": {
{
name: "test2",
environment: "env1",
date: simpleDate(2001, 3),
status: "Failed",
duration: 1,
}, {
name: "test2",
environment: "env1",
date: simpleDate(2001, 2),
status: "Failed",
duration: 3,
}, {
name: "test2",
environment: "env1",
date: simpleDate(2001, 1),
status: "Failed",
duration: 3,
},
},
},
"env2": {
"test2": {
{
name: "test2",
environment: "env2",
date: simpleDate(2003, 3),
status: "Passed",
duration: 0.5,
}, testEntry{
name: "test2",
environment: "env2",
date: simpleDate(2003, 2),
status: "Failed",
duration: 1.5,
},
},
},
})
expectedData := map[string]map[string]float32{
"env1": {
"test1": float32(12) / float32(5),
"test2": float32(7) / float32(3),
},
"env2": {
"test2": 1,
},
}
compareValues(t, actualData, expectedData)
}

2
hack/jenkins/test-flake-chart/report_flakes.sh
View File

@ -54,7 +54,7 @@ TMP_FAILED_RATES="$TMP_FLAKE_RATES\_filtered"
# 3) Join the flake rates with the failing tests to only get flake rates of failing tests.
# 4) Sort failed test flake rates based on the flakiness of that test - stable tests should be first on the list.
# 5) Store in file $TMP_FAILED_RATES.
< "$TMP_FLAKE_RATES" sed -n -r -e "s/$ENVIRONMENT,([a-zA-Z\/_-]*),([.0-9]*)/\1,\2/p" \
< "$TMP_FLAKE_RATES" sed -n -r -e "s/$ENVIRONMENT,([a-zA-Z\/_-]*),([.0-9]*),[.0-9]*/\1,\2/p" \
| sort -t, -k1,1 \
| join -t , -j 1 "$TMP_DATA" - \
| sort -g -t, -k2,2 \