diff --git a/.github/workflows/time-to-k8s-public-chart.yml b/.github/workflows/time-to-k8s-public-chart.yml new file mode 100644 index 0000000000..ff70bc6290 --- /dev/null +++ b/.github/workflows/time-to-k8s-public-chart.yml @@ -0,0 +1,30 @@ +name: "time-to-k8s Public Chart" +on: + workflow_dispatch: + schedule: + # every day at 7am & 7pm pacific + - cron: "0 2,14 * * *" +env: + GOPROXY: https://proxy.golang.org + GO_VERSION: 1.16.5 +jobs: + time-to-k8s-public-chart: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: ${{env.GO_VERSION}} + stable: true + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@master + with: + project_id: ${{ secrets.GCP_PROJECT_ID }} + service_account_key: ${{ secrets.GCP_TIME_TO_K8S_SA_KEY }} + export_default_credentials: true + - name: Benchmark time-to-k8s for Docker + run: | + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh docker + - name: Benchmark time-to-k8s for Containerd + run: | + ./hack/benchmark/time-to-k8s/public-chart/public-chart.sh containerd diff --git a/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml new file mode 100644 index 0000000000..2fc4ff15ca --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/containerd-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --container-runtime=containerd --memory=max --cpus=max + teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml b/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml new file mode 100644 index 0000000000..cfc224e1c2 --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/docker-benchmark.yaml @@ -0,0 +1,4 @@ +testcases: + minikube: + setup: minikube start --container-runtime=docker --memory=max --cpus=max + teardown: minikube delete diff --git a/hack/benchmark/time-to-k8s/public-chart/generate-chart.go b/hack/benchmark/time-to-k8s/public-chart/generate-chart.go new file mode 100644 index 0000000000..95a8a65ab0 --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/generate-chart.go @@ -0,0 +1,219 @@ +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "encoding/csv" + "encoding/json" + "flag" + "image/color" + "io" + "log" + "os" + "strconv" + "time" + + "gonum.org/v1/plot" + "gonum.org/v1/plot/plotter" + "gonum.org/v1/plot/vg" + "gonum.org/v1/plot/vg/draw" +) + +// benchmark contains the duration of the benchmark steps +type benchmark struct { + Date time.Time `json:"date"` + Cmd float64 `json:"cmd"` + API float64 `json:"api"` + K8s float64 `json:"k8s"` + DNSSvc float64 `json:"dnsSvc"` + App float64 `json:"app"` + DNSAns float64 `json:"dnsAns"` + Total float64 `json:"total"` +} + +// benchmarks contains a list of benchmarks, used for storing benchmark results to JSON +type benchmarks struct { + Benchmarks []benchmark `json:"benchmarks"` +} + +func main() { + latestBenchmarkPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result") + chartOutputPath := flag.String("output", "", "path to output the chart to") + pastBenchmarksPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results") + flag.Parse() + + latestBenchmark := readInLatestBenchmark(*latestBenchmarkPath) + pastBenchmarks := readInPastBenchmarks(*pastBenchmarksPath) + pastBenchmarks.Benchmarks = append(pastBenchmarks.Benchmarks, latestBenchmark) + updateRunsFile(pastBenchmarks, *pastBenchmarksPath) + createChart(pastBenchmarks.Benchmarks, *chartOutputPath) +} + +// readInLatestBenchmark reads in the latest benchmark result from a CSV file +func readInLatestBenchmark(latestBenchmarkPath string) benchmark { + f, err := os.Open(latestBenchmarkPath) + if err != nil { + log.Fatal(err) + } + + var cmd, api, k8s, dnsSvc, app, dnsAns float64 + steps := []*float64{&cmd, &api, &k8s, &dnsSvc, &app, &dnsAns} + count := 0 + + r := csv.NewReader(f) + for { + line, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + log.Fatal(err) + } + + // skip the first line of the CSV file + if line[0] == "name" { + continue + } + + values := []float64{} + + // 8-13 contain the benchmark results + for i := 8; i <= 13; i++ { + v, err := strconv.ParseFloat(line[i], 64) + if err != nil { + log.Fatal(err) + } + values = append(values, v) + } + count++ + for i, step := range steps { + *step += values[i] + } + } + + var total float64 + for _, step := range steps { + *step /= float64(count) + total += *step + } + + return benchmark{time.Now(), cmd, api, k8s, dnsSvc, app, dnsAns, total} +} + +// readInPastBenchmarks reads in the past benchmark results from a JSON file +func readInPastBenchmarks(pastBenchmarksPath string) *benchmarks { + data, err := os.ReadFile(pastBenchmarksPath) + if err != nil { + log.Fatal(err) + } + + b := &benchmarks{} + if err := json.Unmarshal(data, b); err != nil { + log.Fatal(err) + } + + return b +} + +// updateRunsFile overwrites the run file with the updated benchmarks list +func updateRunsFile(h *benchmarks, pastRunsPath string) { + b, err := json.Marshal(h) + if err != nil { + log.Fatal(err) + } + + if err := os.WriteFile(pastRunsPath, b, 0600); err != nil { + log.Fatal(err) + } +} + +// createChart creates a time series chart of the benchmarks +func createChart(benchmarks []benchmark, chartOutputPath string) { + n := len(benchmarks) + var cmdXYs, apiXYs, k8sXYs, dnsSvcXYs, appXYs, dnsAnsXYs, totalXYs plotter.XYs + xys := []*plotter.XYs{&cmdXYs, &apiXYs, &k8sXYs, &dnsSvcXYs, &appXYs, &dnsAnsXYs, &totalXYs} + + for _, xy := range xys { + *xy = make(plotter.XYs, n) + } + + for i, b := range benchmarks { + date := float64(b.Date.Unix()) + xyValues := []struct { + xys *plotter.XYs + value float64 + }{ + {&cmdXYs, b.Cmd}, + {&apiXYs, b.API}, + {&k8sXYs, b.K8s}, + {&dnsSvcXYs, b.DNSSvc}, + {&appXYs, b.App}, + {&dnsAnsXYs, b.DNSAns}, + {&totalXYs, b.Total}, + } + for _, xyValue := range xyValues { + xy := &(*xyValue.xys)[i] + xy.Y = xyValue.value + xy.X = date + } + } + + p := plot.New() + p.Add(plotter.NewGrid()) + p.Legend.Top = true + p.Title.Text = "time-to-k8s" + p.X.Label.Text = "date" + p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"} + p.Y.Label.Text = "time (seconds)" + p.Y.Max = 95 + + steps := []struct { + xys plotter.XYs + rgba color.RGBA + label string + }{ + {cmdXYs, color.RGBA{R: 255, A: 255}, "Command Exec"}, + {apiXYs, color.RGBA{G: 255, A: 255}, "API Server Answering"}, + {k8sXYs, color.RGBA{B: 255, A: 255}, "Kubernetes SVC"}, + {dnsSvcXYs, color.RGBA{R: 255, B: 255, A: 255}, "DNS SVC"}, + {appXYs, color.RGBA{R: 255, G: 255, A: 255}, "App Running"}, + {dnsAnsXYs, color.RGBA{G: 255, B: 255, A: 255}, "DNS Answering"}, + {totalXYs, color.RGBA{B: 255, R: 140, A: 255}, "Total"}, + } + + for _, step := range steps { + line, points := newLinePoints(step.xys, step.rgba) + p.Add(line, points) + p.Legend.Add(step.label, line) + } + + if err := p.Save(12*vg.Inch, 8*vg.Inch, chartOutputPath); err != nil { + log.Fatal(err) + } +} + +func newLinePoints(xys plotter.XYs, lineColor color.RGBA) (*plotter.Line, *plotter.Scatter) { + line, points, err := plotter.NewLinePoints(xys) + if err != nil { + log.Fatal(err) + } + line.Color = lineColor + points.Color = lineColor + points.Shape = draw.CircleGlyph{} + + return line, points +} diff --git a/hack/benchmark/time-to-k8s/public-chart/public-chart.sh b/hack/benchmark/time-to-k8s/public-chart/public-chart.sh new file mode 100755 index 0000000000..5bffd172c0 --- /dev/null +++ b/hack/benchmark/time-to-k8s/public-chart/public-chart.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# container-runtime (docker or containerd) +RUNTIME="$1" + +install_minikube() { + make + sudo install ./out/minikube /usr/local/bin/minikube +} + +run_benchmark() { + ( cd ./hack/benchmark/time-to-k8s/time-to-k8s-repo/ && + git submodule update --init && + go run . --config "../public-chart/$RUNTIME-benchmark.yaml" --iterations 10 --output ./output.csv ) +} + +generate_chart() { + go run ./hack/benchmark/time-to-k8s/public-chart/generate-chart.go --csv ./hack/benchmark/time-to-k8s/time-to-k8s-repo/output.csv --output ./chart.png --past-runs ./runs.json +} + +cleanup() { + rm ./runs.json + rm ./hack/benchmark/time-to-k8s/time-to-k8s-repo/output.csv + rm ./chart.png +} + +gsutil -m cp "gs://minikube-time-to-k8s/$RUNTIME-runs.json" ./runs.json + +install_minikube + +run_benchmark +generate_chart + +gsutil -m cp ./runs.json "gs://minikube-time-to-k8s/$RUNTIME-runs.json" +gsutil -m cp ./runs.json "gs://minikube-time-to-k8s/$(date +'%Y-%m-%d')-$RUNTIME.json" +gsutil -m cp ./chart.png "gs://minikube-time-to-k8s/$RUNTIME-chart.png" + +cleanup diff --git a/hack/benchmark/time-to-k8s/time-to-k8s.sh b/hack/benchmark/time-to-k8s/time-to-k8s.sh index cdce28dee4..f632ce8fd0 100755 --- a/hack/benchmark/time-to-k8s/time-to-k8s.sh +++ b/hack/benchmark/time-to-k8s/time-to-k8s.sh @@ -32,7 +32,6 @@ install_minikube() { } run_benchmark() { - pwd ( cd ./hack/benchmark/time-to-k8s/time-to-k8s-repo/ && git submodule update --init && go run . --config local-kubernetes.yaml --iterations 10 --output output.csv ) diff --git a/hack/update/golang_version/update_golang_version.go b/hack/update/golang_version/update_golang_version.go index 8e99d221c7..15ec5691db 100644 --- a/hack/update/golang_version/update_golang_version.go +++ b/hack/update/golang_version/update_golang_version.go @@ -90,6 +90,11 @@ var ( `GO_VERSION: '.*`: `GO_VERSION: '{{.StableVersion}}'`, }, }, + ".github/workflows/time-to-k8s-public-chart.yaml": { + Replace: map[string]string{ + `GO_VERSION: '.*`: `GO_VERSION: '{{.StableVersion}}'`, + }, + }, ".travis.yml": { Replace: map[string]string{ `go:\n - .*`: `go:{{printf "\n - %s" .StableVersion}}`, diff --git a/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md b/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md new file mode 100644 index 0000000000..c65295ee01 --- /dev/null +++ b/site/content/en/docs/benchmarks/timeToK8s/daily_benchmark.md @@ -0,0 +1,16 @@ +--- +title: "Daily Benchmark" +description: > + Chart to visualize the time-to-k8s benchmark daily against HEAD +weight: -99999999 +--- + +[Benchmarking Machine Specs](https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources) + +## Docker + +![Docker Benchmarks](https://storage.googleapis.com/minikube-time-to-k8s/docker-chart.png) + +## Containerd + +![Containerd Benchmarks](https://storage.googleapis.com/minikube-time-to-k8s/containerd-chart.png)