Add support for specifing pr when running mkcmp

This PR adds support for running:

./out/mkcmp pr://1000

which downloads the minikube binary generated by Jenkins for PR 1000 before running mkcmp.

cmd/performance/mkcmp/cmd/mkcmp.go

@@ -35,7 +35,12 @@ var rootCmd = &cobra.Command{
 		return validateArgs(args)
 	},
 	RunE: func(cmd *cobra.Command, args []string) error {
-		return perf.CompareMinikubeStart(context.Background(), os.Stdout, args)
+		binaries, err := retrieveBinaries(args)
+		if err != nil {
+			return err
+		}
+		perf.CompareMinikubeStart(context.Background(), os.Stdout, binaries)
+		return nil
 	},
 }
 
@@ -46,6 +51,18 @@ func validateArgs(args []string) error {
 	return nil
 }
 
+func retrieveBinaries(args []string) ([]*perf.Binary, error) {
+	binaries := []*perf.Binary{}
+	for _, a := range args {
+		binary, err := perf.NewBinary(a)
+		if err != nil {
+			return nil, err
+		}
+		binaries = append(binaries, binary)
+	}
+	return binaries, nil
+}
+
 // Execute runs the mkcmp command
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {

pkg/minikube/perf/binary.go

@@ -0,0 +1,105 @@
+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package perf
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/constants"
+)
+
+type Binary struct {
+	path string
+	pr   int
+}
+
+const (
+	prPrefix = "pr://"
+)
+
+// NewBinary returns a new binary type
+func NewBinary(b string) (*Binary, error) {
+	// If it doesn't have the prefix, assume a path
+	if !strings.HasPrefix(b, prPrefix) {
+		return &Binary{
+			path: b,
+		}, nil
+	}
+	return newBinaryFromPR(b)
+}
+
+// Name returns the name of the binary
+func (b *Binary) Name() string {
+	if b.pr != 0 {
+		return fmt.Sprintf("Minikube (PR %d)", b.pr)
+	}
+	return filepath.Base(b.path)
+}
+
+// newBinaryFromPR downloads the minikube binary built for the pr by Jenkins from GCS
+func newBinaryFromPR(pr string) (*Binary, error) {
+	pr = strings.TrimPrefix(pr, prPrefix)
+	// try to convert to int
+	i, err := strconv.Atoi(pr)
+	if err != nil {
+		return nil, errors.Wrapf(err, "converting %s to an integer", pr)
+	}
+
+	b := &Binary{
+		path: localMinikubePath(i),
+		pr:   i,
+	}
+
+	if err := downloadBinary(remoteMinikubeURL(i), b.path); err != nil {
+		return nil, errors.Wrapf(err, "downloading minikube")
+	}
+
+	return b, nil
+}
+
+func remoteMinikubeURL(pr int) string {
+	return fmt.Sprintf("https://storage.googleapis.com/minikube-builds/%d/minikube-linux-amd64", pr)
+}
+
+func localMinikubePath(pr int) string {
+	return fmt.Sprintf("%s/minikube-binaries/%d/minikube", constants.DefaultMinipath, pr)
+}
+
+func downloadBinary(url, path string) error {
+	resp, err := http.Get(url)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0777)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	_, err = io.Copy(f, resp.Body)
+	return err
+}

pkg/minikube/perf/start.go

@@ -39,17 +39,22 @@ var (
 )
 
 // CompareMinikubeStart compares the time to run `minikube start` between two minikube binaries
-func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []string) error {
+func CompareMinikubeStart(ctx context.Context, out io.Writer, binaries []*Binary) error {
 	durations, err := collectTimes(ctx, binaries)
 	if err != nil {
 		return err
 	}
 
-	fmt.Fprintf(out, "Old binary: %v\nNew binary: %v\nAverage Old: %f\nAverage New: %f\n", durations[0], durations[1], average(durations[0]), average(durations[1]))
+	for i, d := range durations {
+		fmt.Printf("Results for %s:\n", binaries[i].Name())
+		fmt.Printf("Times: %v\n", d)
+		fmt.Printf("Average Time: %f\n\n", average(d))
+	}
+
 	return nil
 }
 
-func collectTimes(ctx context.Context, binaries []string) ([][]float64, error) {
+func collectTimes(ctx context.Context, binaries []*Binary) ([][]float64, error) {
 	durations := make([][]float64, len(binaries))
 	for i := range durations {
 		durations[i] = make([]float64, runs)
@@ -79,12 +84,12 @@ func average(nums []float64) float64 {
 
 // timeMinikubeStart returns the time it takes to execute `minikube start`
 // It deletes the VM after `minikube start`.
-func timeMinikubeStart(ctx context.Context, binary string) (float64, error) {
-	startCmd := exec.CommandContext(ctx, binary, "start")
+func timeMinikubeStart(ctx context.Context, binary *Binary) (float64, error) {
+	startCmd := exec.CommandContext(ctx, binary.path, "start")
 	startCmd.Stdout = os.Stdout
 	startCmd.Stderr = os.Stderr
 
-	deleteCmd := exec.CommandContext(ctx, binary, "delete")
+	deleteCmd := exec.CommandContext(ctx, binary.path, "delete")
 	defer func() {
 		if err := deleteCmd.Run(); err != nil {
 			log.Printf("error deleting minikube: %v", err)