add github action for running minikube-image-benchmark
Товарищ программист
Raiden Shogun
parent
8cdc5c73a5
commit
0fcd811253
|
|
@ -0,0 +1,31 @@
|
|||
name: "publish image benchmark"
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
# every day at 7am & 7pm pacific
|
||||
- cron: "0 2,14 * * *"
|
||||
env:
|
||||
GOPROXY: https://proxy.golang.org
|
||||
GO_VERSION: '1.20.6'
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
image-benchmark:
|
||||
if: github.repository == 'kubernetes/minikube'
|
||||
runs-on: ubuntu-20.04
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: 'us-west-1'
|
||||
steps:
|
||||
- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
|
||||
- uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753
|
||||
with:
|
||||
go-version: ${{env.GO_VERSION}}
|
||||
cache-dependency-path: ./go.sum
|
||||
- name: Run Benchmark
|
||||
run: |
|
||||
./hack/benchmark/image-build/publish-chart.sh
|
||||
|
||||
|
||||
|
|
@ -4,3 +4,6 @@
|
|||
[submodule "hack/benchmark/time-to-k8s/time-to-k8s-repo"]
|
||||
path = hack/benchmark/time-to-k8s/time-to-k8s-repo
|
||||
url = https://github.com/tstromberg/time-to-k8s.git
|
||||
[submodule "hack/benchmark/image-build/minikube-image-benchmark"]
|
||||
path = hack/benchmark/image-build/minikube-image-benchmark
|
||||
url = https://github.com/GoogleContainerTools/minikube-image-benchmark.git
|
||||
|
|
|
|||
|
|
@ -0,0 +1,280 @@
|
|||
/*
|
||||
Copyright 2023 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image/color"
|
||||
"io"
|
||||
"log"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"gonum.org/v1/plot"
|
||||
"gonum.org/v1/plot/plotter"
|
||||
"gonum.org/v1/plot/plotutil"
|
||||
"gonum.org/v1/plot/vg"
|
||||
"gonum.org/v1/plot/vg/draw"
|
||||
)
|
||||
|
||||
var Images = []string{
|
||||
"buildpacksFewLargeFiles",
|
||||
// to simplify the output, the following images are omitted
|
||||
// "buildpacksFewSmallFiles",
|
||||
// "buildpacksManyLargeFiles",
|
||||
// "buildpacksManySmallFiles",
|
||||
}
|
||||
|
||||
var Environments = []string{
|
||||
"MinikubeImageLoadDocker",
|
||||
"MinikubeImageBuild",
|
||||
"MinikubeDockerEnvDocker",
|
||||
"MinikubeAddonRegistryDocker",
|
||||
"MinikubeImageLoadContainerd",
|
||||
"MinikubeImageContainerd",
|
||||
"MinikubeAddonRegistryContainerd",
|
||||
"MinikubeImageLoadCrio",
|
||||
"MinikubeImageCrio",
|
||||
"MinikubeAddonRegistryCrio",
|
||||
"Kind",
|
||||
"K3d",
|
||||
"Microk8s",
|
||||
}
|
||||
|
||||
var RuntimeEnvironments = map[string][]string{
|
||||
"docker": {
|
||||
"MinikubeImageLoadDocker",
|
||||
"MinikubeImageBuild",
|
||||
"MinikubeDockerEnvDocker",
|
||||
"MinikubeAddonRegistryDocker",
|
||||
},
|
||||
|
||||
"containerd": {
|
||||
"MinikubeImageLoadContainerd",
|
||||
"MinikubeImageContainerd",
|
||||
"MinikubeAddonRegistryContainerd",
|
||||
},
|
||||
}
|
||||
|
||||
const (
|
||||
INTERATIVE = "Iterative"
|
||||
NONINTERATIVE = "NonIterative"
|
||||
)
|
||||
|
||||
var Methods = []string{
|
||||
INTERATIVE,
|
||||
// to simplify the output, non-interative is omitted
|
||||
// NONINTERATIVE,
|
||||
}
|
||||
|
||||
// env name-> test result
|
||||
type TestResult map[string]float64
|
||||
|
||||
func NewTestResult(values []float64) TestResult {
|
||||
res := make(TestResult)
|
||||
for index, v := range values {
|
||||
res[Environments[index]] = v
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
// imageName->TestResult
|
||||
type ImageTestResults map[string]TestResult
|
||||
|
||||
type MethodTestResults struct {
|
||||
Date time.Time
|
||||
// method name -> results
|
||||
Results map[string]ImageTestResults
|
||||
}
|
||||
|
||||
type Records struct {
|
||||
Records []MethodTestResults
|
||||
}
|
||||
|
||||
func main() {
|
||||
latestTestResultPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result")
|
||||
pastTestRecordsPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results")
|
||||
chartsPath := flag.String("charts", "", "path to the folder to write the daily charts to")
|
||||
flag.Parse()
|
||||
|
||||
latestBenchmark := readInLatestTestResult(*latestTestResultPath)
|
||||
latestBenchmark.Date = time.Now()
|
||||
pastBenchmarks := readInPastTestResults(*pastTestRecordsPath)
|
||||
pastBenchmarks.Records = append(pastBenchmarks.Records, latestBenchmark)
|
||||
updatePastTestResults(pastBenchmarks, *pastTestRecordsPath)
|
||||
createDailyChart(pastBenchmarks, *chartsPath)
|
||||
}
|
||||
|
||||
// readInLatestTestResult reads in the latest benchmark result from a CSV file
|
||||
// and return the MethodTestResults object
|
||||
func readInLatestTestResult(latestBenchmarkPath string) MethodTestResults {
|
||||
|
||||
var res = MethodTestResults{
|
||||
Results: make(map[string]ImageTestResults),
|
||||
}
|
||||
res.Results[INTERATIVE] = make(ImageTestResults)
|
||||
res.Results[NONINTERATIVE] = make(ImageTestResults)
|
||||
|
||||
f, err := os.Open(latestBenchmarkPath)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
r := csv.NewReader(f)
|
||||
for {
|
||||
line, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// skip the first line of the CSV file
|
||||
if line[0] == "image" {
|
||||
continue
|
||||
}
|
||||
|
||||
valuesInterative := []float64{}
|
||||
valuesNonInterative := []float64{}
|
||||
// interative test results of each env are stored in the following columns
|
||||
indicesInterative := []int{1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49}
|
||||
// non-interative test results of each env are stored in the following columns
|
||||
indicesNonInterative := []int{3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51}
|
||||
|
||||
for _, i := range indicesInterative {
|
||||
v, err := strconv.ParseFloat(line[i], 64)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
valuesInterative = append(valuesInterative, v)
|
||||
}
|
||||
|
||||
for _, i := range indicesNonInterative {
|
||||
v, err := strconv.ParseFloat(line[i], 64)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
valuesNonInterative = append(valuesNonInterative, v)
|
||||
}
|
||||
|
||||
imageName := line[0]
|
||||
|
||||
res.Results[INTERATIVE][imageName] = NewTestResult(valuesInterative)
|
||||
res.Results[NONINTERATIVE][imageName] = NewTestResult(valuesNonInterative)
|
||||
|
||||
}
|
||||
|
||||
return res
|
||||
}
|
||||
|
||||
// readInPastTestResults reads in the past benchmark results from a JSON file
|
||||
func readInPastTestResults(pastTestRecordPath string) Records {
|
||||
|
||||
record := Records{}
|
||||
data, err := os.ReadFile(pastTestRecordPath)
|
||||
if os.IsNotExist(err) {
|
||||
return record
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(data, &record); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
return record
|
||||
}
|
||||
|
||||
// updateRunsFile overwrites the run file with the updated benchmarks list
|
||||
func updatePastTestResults(h Records, pastTestRecordPath string) {
|
||||
b, err := json.Marshal(h)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(pastTestRecordPath, b, 0600); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
func createDailyChart(record Records, outputFolder string) {
|
||||
|
||||
for _, method := range Methods {
|
||||
for _, image := range Images {
|
||||
createChart(record, method, image, "docker", outputFolder)
|
||||
createChart(record, method, image, "containerd", outputFolder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func createChart(record Records, methodName string, imageName string, runtime string, chartOutputPath string) {
|
||||
p := plot.New()
|
||||
p.Add(plotter.NewGrid())
|
||||
p.Legend.Top = true
|
||||
p.Title.Text = fmt.Sprintf("%s-%s-%s-performance", methodName, imageName, runtime)
|
||||
p.X.Label.Text = "date"
|
||||
p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"}
|
||||
p.Y.Label.Text = "time (seconds)"
|
||||
yMaxTotal := float64(0)
|
||||
|
||||
// gonum plot do not have enough default colors in any group
|
||||
// so we combine different group of default colors
|
||||
colors := append([]color.Color{}, plotutil.SoftColors...)
|
||||
colors = append(colors, plotutil.DarkColors...)
|
||||
|
||||
pointGroup := make(map[string]plotter.XYs)
|
||||
for _, name := range RuntimeEnvironments[runtime] {
|
||||
pointGroup[name] = make(plotter.XYs, len(record.Records))
|
||||
|
||||
}
|
||||
|
||||
for i := 0; i < len(record.Records); i++ {
|
||||
for _, envName := range RuntimeEnvironments[runtime] {
|
||||
pointGroup[envName][i].X = float64(record.Records[i].Date.Unix())
|
||||
pointGroup[envName][i].Y = record.Records[i].Results[methodName][imageName][envName]
|
||||
yMaxTotal = math.Max(yMaxTotal, pointGroup[envName][i].Y)
|
||||
}
|
||||
}
|
||||
p.Y.Max = yMaxTotal
|
||||
|
||||
i := 0
|
||||
for envName, xys := range pointGroup {
|
||||
line, points, err := plotter.NewLinePoints(xys)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
line.Color = colors[i]
|
||||
points.Color = colors[i]
|
||||
points.Shape = draw.CircleGlyph{}
|
||||
i++
|
||||
p.Add(line, points)
|
||||
p.Legend.Add(envName, line)
|
||||
}
|
||||
|
||||
filename := filepath.Join(chartOutputPath, fmt.Sprintf("%s_%s_%s_chart.png", methodName, imageName, runtime))
|
||||
|
||||
if err := p.Save(12*vg.Inch, 8*vg.Inch, filename); err != nil {
|
||||
log.Fatalf("failed creating png: %v", err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit feab1337c92e1cd01d29e24c085407ec5ebdc3d2
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2023 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -x
|
||||
|
||||
BUCKET="s3://image-benchmark"
|
||||
|
||||
install_minikube() {
|
||||
make
|
||||
sudo install ./out/minikube /usr/local/bin/minikube
|
||||
}
|
||||
|
||||
run_benchmark() {
|
||||
( cd ./hack/benchmark/image-build/minikube-image-benchmark &&
|
||||
git submodule update --init &&
|
||||
make &&
|
||||
./out/benchmark )
|
||||
}
|
||||
|
||||
generate_chart() {
|
||||
go run ./hack/benchmark/image-build/generate-chart.go --csv hack/benchmark/image-build/minikube-image-benchmark/out/results.csv --past-runs record.json
|
||||
}
|
||||
|
||||
copy() {
|
||||
aws s3 cp "$1" "$2"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
rm ./Iterative_buildpacksFewLargeFiles_containerd_chart.png
|
||||
rm ./Iterative_buildpacksFewLargeFiles_docker_chart.png
|
||||
rm hack/benchmark/image-build/minikube-image-benchmark/out/results.csv
|
||||
}
|
||||
|
||||
|
||||
install_minikube
|
||||
copy "$BUCKET/record.json" ./record.json
|
||||
set -e
|
||||
|
||||
run_benchmark
|
||||
generate_chart
|
||||
|
||||
copy ./record.json "$BUCKET/record.json"
|
||||
copy ./Iterative_buildpacksFewLargeFiles_containerd_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_containerd_chart.png"
|
||||
copy ./Iterative_buildpacksFewLargeFiles_docker_chart.png "$BUCKET/Iterative_buildpacksFewLargeFiles_docker_chart.png"
|
||||
cleanup
|
||||
Loading…
Reference in New Issue