minikube/hack/benchmark/image-build/generate-chart.go

/*
Copyright 2023 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"image/color"
	"io"
	"log"
	"math"
	"os"
	"path/filepath"
	"strconv"
	"time"

	"gonum.org/v1/plot"
	"gonum.org/v1/plot/plotter"
	"gonum.org/v1/plot/plotutil"
	"gonum.org/v1/plot/vg"
	"gonum.org/v1/plot/vg/draw"
)

var Images = []string{
	"buildpacksFewLargeFiles",
	// to simplify the output, the following images are omitted
	// "buildpacksFewSmallFiles",
	// "buildpacksManyLargeFiles",
	// "buildpacksManySmallFiles",
}

var Environments = []string{
	"MinikubeImageLoadDocker",
	"MinikubeImageBuild",
	"MinikubeDockerEnvDocker",
	"MinikubeAddonRegistryDocker",
	"MinikubeImageLoadContainerd",
	"MinikubeImageContainerd",
	"MinikubeDockerEnvContainerd",
	"MinikubeAddonRegistryContainerd",
	"MinikubeImageLoadCrio",
	"MinikubeImageCrio",
	"MinikubeAddonRegistryCrio",
	"Kind",
	"K3d",
	"Microk8s",
}

var RuntimeMethods = map[string][]string{
	"docker": {
		"MinikubeImageLoadDocker",
		"MinikubeImageBuild",
		"MinikubeDockerEnvDocker",
		"MinikubeAddonRegistryDocker",
	},

	"containerd": {
		"MinikubeImageLoadContainerd",
		"MinikubeImageContainerd",
		"MinikubeDockerEnvContainerd",
		"MinikubeAddonRegistryContainerd",
	},
}

const (
	INTERATIVE    = "Iterative"
	NONINTERATIVE = "NonIterative"
)

var Itrs = []string{
	INTERATIVE,
	// to simplify the output, non-interative is omitted
	// NONINTERATIVE,
}

// method name-> test result
type TestResult map[string]float64

func NewTestResult(values []float64) TestResult {
	res := make(TestResult)
	for index, v := range values {
		res[Environments[index]] = v
	}
	return res
}

// imageName->TestResult
type ImageTestResults map[string]TestResult

type ItrTestResults struct {
	Date time.Time
	// itr name -> results
	Results map[string]ImageTestResults
}

type Records struct {
	Records []ItrTestResults
}

func main() {
	latestTestResultPath := flag.String("csv", "", "path to the CSV file containing the latest benchmark result")
	pastTestRecordsPath := flag.String("past-runs", "", "path to the JSON file containing the past benchmark results")
	chartsPath := flag.String("charts", "", "path to the folder to write the daily charts to")
	flag.Parse()

	latestBenchmark := readInLatestTestResult(*latestTestResultPath)
	latestBenchmark.Date = time.Now()
	pastBenchmarks := readInPastTestResults(*pastTestRecordsPath)
	pastBenchmarks.Records = append(pastBenchmarks.Records, latestBenchmark)
	updatePastTestResults(pastBenchmarks, *pastTestRecordsPath)
	createDailyChart(pastBenchmarks, *chartsPath)
}

// readInLatestTestResult reads in the latest benchmark result from a CSV file
// and return the MethodTestResults object
func readInLatestTestResult(latestBenchmarkPath string) ItrTestResults {

	var res = ItrTestResults{
		Results: make(map[string]ImageTestResults),
	}
	res.Results[INTERATIVE] = make(ImageTestResults)
	res.Results[NONINTERATIVE] = make(ImageTestResults)

	f, err := os.Open(latestBenchmarkPath)
	if err != nil {
		log.Fatal(err)
	}

	r := csv.NewReader(f)
	for {
		line, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			log.Fatal(err)
		}

		// skip the first line of the CSV file
		if line[0] == "image" {
			continue
		}

		valuesInterative := []float64{}
		valuesNonInterative := []float64{}
		// interative test results of each env are stored in the following columns
		indicesInterative := []int{1, 5, 9, 13, 17, 21, 25, 29, 33, 37, 41, 45, 49, 53}
		// non-interative test results of each env are stored in the following columns
		indicesNonInterative := []int{3, 7, 11, 15, 19, 23, 27, 31, 35, 39, 43, 47, 51, 55}

		for _, i := range indicesInterative {
			if line[i] == "NaN" {
				// we use -1 as invalid value
				valuesInterative = append(valuesInterative, -1)
				continue
			}
			v, err := strconv.ParseFloat(line[i], 64)
			if err != nil {
				log.Fatal(err)
			}
			valuesInterative = append(valuesInterative, v)
		}

		for _, i := range indicesNonInterative {
			if line[i] == "NaN" {
				// we use -1 as invalid value
				valuesNonInterative = append(valuesNonInterative, -1)
				continue
			}
			v, err := strconv.ParseFloat(line[i], 64)
			if err != nil {
				log.Fatal(err)
			}
			valuesNonInterative = append(valuesNonInterative, v)
		}

		imageName := line[0]

		res.Results[INTERATIVE][imageName] = NewTestResult(valuesInterative)
		res.Results[NONINTERATIVE][imageName] = NewTestResult(valuesNonInterative)

	}

	return res
}

// readInPastTestResults reads in the past benchmark results from a JSON file
func readInPastTestResults(pastTestRecordPath string) Records {

	record := Records{}
	data, err := os.ReadFile(pastTestRecordPath)
	if os.IsNotExist(err) {
		return record
	}
	if err != nil {
		log.Fatal(err)
	}

	if err := json.Unmarshal(data, &record); err != nil {
		log.Fatal(err)
	}

	return record
}

// updatePastTestResults overwrites the run file with the updated benchmarks list
func updatePastTestResults(h Records, pastTestRecordPath string) {
	b, err := json.Marshal(h)
	if err != nil {
		log.Fatal(err)
	}

	if err := os.WriteFile(pastTestRecordPath, b, 0600); err != nil {
		log.Fatal(err)
	}
}
func createDailyChart(record Records, outputFolder string) {

	for _, itr := range Itrs {
		for _, image := range Images {
			createChart(record, itr, image, "docker", outputFolder)
			createChart(record, itr, image, "containerd", outputFolder)
		}
	}
}

func createChart(record Records, itr string, imageName string, runtime string, chartOutputPath string) {
	p := plot.New()
	p.Add(plotter.NewGrid())
	p.Legend.Top = true
	p.Title.Text = fmt.Sprintf("%s-%s-%s-performance", itr, imageName, runtime)
	p.X.Label.Text = "date"
	p.X.Tick.Marker = plot.TimeTicks{Format: "2006-01-02"}
	p.Y.Label.Text = "time (seconds)"
	yMaxTotal := float64(0)

	// gonum plot do not have enough default colors in any group
	// so we combine different group of default colors
	colors := append([]color.Color{}, plotutil.SoftColors...)
	colors = append(colors, plotutil.DarkColors...)

	pointGroup := make(map[string]plotter.XYs)
	for _, name := range RuntimeMethods[runtime] {
		pointGroup[name] = make(plotter.XYs, 0)
	}

	for i := 0; i < len(record.Records); i++ {
		for _, method := range RuntimeMethods[runtime] {
			// for invalid values(<0) this point is dropped
			if v, ok := record.Records[i].Results[itr][imageName][method]; ok && v > 0 {
				point := plotter.XY{
					X: float64(record.Records[i].Date.Unix()),
					Y: record.Records[i].Results[itr][imageName][method],
				}
				pointGroup[method] = append(pointGroup[method], point)

				yMaxTotal = math.Max(yMaxTotal, point.Y)

			}
		}
	}
	p.Y.Max = yMaxTotal

	i := 0
	for method, xys := range pointGroup {
		line, points, err := plotter.NewLinePoints(xys)
		if err != nil {
			log.Fatal(err)
		}
		line.Color = colors[i]
		points.Color = colors[i]
		points.Shape = draw.CircleGlyph{}
		i++
		p.Add(line, points)
		p.Legend.Add(method, line)
	}

	filename := filepath.Join(chartOutputPath, fmt.Sprintf("%s_%s_%s_chart.png", itr, imageName, runtime))

	if err := p.Save(12*vg.Inch, 8*vg.Inch, filename); err != nil {
		log.Fatalf("failed creating png: %v", err)
	}
}