From 96ac580effad4d9c4be45d91363b67a94cfda9f6 Mon Sep 17 00:00:00 2001 From: prezha Date: Wed, 14 Oct 2020 01:44:08 +0100 Subject: [PATCH] update: consolidation & automation --- hack/update/filesystem.go | 53 ++++ hack/update/github.go | 267 ++++++++++++++++++ .../golang_version/update_golang_version.go | 132 +++++++++ .../kicbase_version/update_kicbase_version.go | 148 ++++++++++ .../update_kubernetes_version.go | 84 ++++++ hack/update/registry.go | 134 +++++++++ hack/update/update.go | 212 ++++++++++++++ 7 files changed, 1030 insertions(+) create mode 100644 hack/update/filesystem.go create mode 100644 hack/update/github.go create mode 100644 hack/update/golang_version/update_golang_version.go create mode 100644 hack/update/kicbase_version/update_kicbase_version.go create mode 100644 hack/update/kubernetes_version/update_kubernetes_version.go create mode 100644 hack/update/registry.go create mode 100644 hack/update/update.go diff --git a/hack/update/filesystem.go b/hack/update/filesystem.go new file mode 100644 index 0000000000..037c0ffcd6 --- /dev/null +++ b/hack/update/filesystem.go @@ -0,0 +1,53 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// fsUpdate updates local filesystem repo files according to the given schema and data, +// returns if the update actually changed anything, and any error occurred +func fsUpdate(fsRoot string, schema map[string]Item, data interface{}) (changed bool, err error) { + for path, item := range schema { + path = filepath.Join(fsRoot, path) + blob, err := ioutil.ReadFile(path) + if err != nil { + return false, err + } + info, err := os.Stat(path) + if err != nil { + return false, err + } + mode := info.Mode() + + item.Content = blob + chg, err := item.apply(data) + if err != nil { + return false, err + } + if chg { + changed = true + } + if err := ioutil.WriteFile(path, item.Content, mode); err != nil { + return false, err + } + } + return changed, nil +} diff --git a/hack/update/github.go b/hack/update/github.go new file mode 100644 index 0000000000..bc82e2ea1d --- /dev/null +++ b/hack/update/github.go @@ -0,0 +1,267 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "golang.org/x/oauth2" + + "github.com/google/go-github/v32/github" + "k8s.io/klog/v2" +) + +const ( + // ghListPerPage uses max value (100) for PerPage to avoid hitting the rate limits + // (ref: https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting) + ghListPerPage = 100 + + // ghSearchLimit limits the number of searched items to be <= N * ListPerPage + ghSearchLimit = 100 +) + +var ( + // GitHub repo data + ghToken = os.Getenv("GITHUB_TOKEN") + ghOwner = "kubernetes" + ghRepo = "minikube" + ghBase = "master" // could be "main" in the future? +) + +// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head +// commit fork, as defined by the schema and data, and also returns any error occurred +// PR branch will be named by the branch, sufixed by '_' and first 7 characters of fork commit SHA +// PR itself will be named by the title and will reference the issue +func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, issue int, token string, schema map[string]Item, data interface{}) (*github.PullRequest, error) { + ghc := ghClient(ctx, token) + + // get base branch + baseBranch, _, err := ghc.Repositories.GetBranch(ctx, owner, repo, base) + if err != nil { + return nil, fmt.Errorf("error getting base branch: %w", err) + } + + // get base commit + baseCommit, _, err := ghc.Repositories.GetCommit(ctx, owner, repo, *baseBranch.Commit.SHA) + if err != nil { + return nil, fmt.Errorf("error getting base commit: %w", err) + } + + // get base tree + baseTree, _, err := ghc.Git.GetTree(ctx, owner, repo, baseCommit.GetSHA(), true) + if err != nil { + return nil, fmt.Errorf("error getting base tree: %w", err) + } + + // update files + changes, err := ghUpdate(ctx, owner, repo, baseTree, token, schema, data) + if err != nil { + return nil, fmt.Errorf("error updating files: %w", err) + } + if changes == nil { + return nil, nil + } + + // create fork + fork, resp, err := ghc.Repositories.CreateFork(ctx, owner, repo, nil) + // https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork + // This method might return an *AcceptedError and a status code of 202. This is because this is + // the status that GitHub returns to signify that it is now computing creating the fork in a + // background task. In this event, the Repository value will be returned, which includes the + // details about the pending fork. A follow up request, after a delay of a second or so, should + // result in a successful request. + if resp.StatusCode == 202 { // *AcceptedError + time.Sleep(time.Second * 5) + } else if err != nil { + return nil, fmt.Errorf("error creating fork: %w", err) + } + + // create fork tree from base and changed files + forkTree, _, err := ghc.Git.CreateTree(ctx, *fork.Owner.Login, *fork.Name, *baseTree.SHA, changes) + if err != nil { + return nil, fmt.Errorf("error creating fork tree: %w", err) + } + + // create fork commit + forkCommit, _, err := ghc.Git.CreateCommit(ctx, *fork.Owner.Login, *fork.Name, &github.Commit{ + Message: github.String(title), + Tree: &github.Tree{SHA: forkTree.SHA}, + Parents: []*github.Commit{{SHA: baseCommit.SHA}}, + }) + if err != nil { + return nil, fmt.Errorf("error creating fork commit: %w", err) + } + klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) + + // create PR branch + prBranch := branch + forkCommit.GetSHA()[:7] + prRef, _, err := ghc.Git.CreateRef(ctx, *fork.Owner.Login, *fork.Name, &github.Reference{ + Ref: github.String("refs/heads/" + prBranch), + Object: &github.GitObject{ + Type: github.String("commit"), + SHA: forkCommit.SHA, + }, + }) + if err != nil { + return nil, fmt.Errorf("error creating PR branch: %w", err) + } + klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) + + // create PR + plan, err := GetPlan(schema, data) + if err != nil { + klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + } + modifiable := true + pr, _, err := ghc.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{ + Title: github.String(title), + Head: github.String(*fork.Owner.Login + ":" + prBranch), + Base: github.String(base), + Body: github.String(fmt.Sprintf("fixes #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, plan)), + MaintainerCanModify: &modifiable, + }) + if err != nil { + return nil, fmt.Errorf("error creating pull request: %w", err) + } + return pr, nil +} + +// ghUpdate updates remote GitHub owner/repo tree according to the given token, schema and data, +// returns resulting changes, and any error occurred +func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, schema map[string]Item, data interface{}) (changes []*github.TreeEntry, err error) { + ghc := ghClient(ctx, token) + + // load each schema item content and update it creating new GitHub TreeEntries + cnt := len(schema) // expected number of files to change + for _, org := range tree.Entries { + if *org.Type == "blob" { + if item, match := schema[*org.Path]; match { + blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA) + if err != nil { + return nil, fmt.Errorf("error getting file: %w", err) + } + item.Content = blob + changed, err := item.apply(data) + if err != nil { + return nil, fmt.Errorf("error updating file: %w", err) + } + if changed { + // add github.TreeEntry that will replace original path content with updated one + changes = append(changes, &github.TreeEntry{ + Path: org.Path, + Mode: org.Mode, + Type: org.Type, + Content: github.String(string(item.Content)), + }) + } + if cnt--; cnt == 0 { + break + } + } + } + } + if cnt != 0 { + return nil, fmt.Errorf("error finding all the files (%d missing) - check the Plan: %w", cnt, err) + } + return changes, nil +} + +// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred +func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url string, err error) { + ghc := ghClient(ctx, token) + + // walk through the paginated list of all pull requests, from latest to older releases + opts := &github.PullRequestListOptions{State: "all", Base: base, ListOptions: github.ListOptions{PerPage: ghListPerPage}} + for (opts.Page+1)*ghListPerPage <= ghSearchLimit { + prs, resp, err := ghc.PullRequests.List(ctx, owner, repo, opts) + if err != nil { + return "", err + } + for _, pr := range prs { + if pr.GetTitle() == title { + return pr.GetHTMLURL(), nil + } + } + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return "", nil +} + +// ghClient returns GitHub Client with a given context and optional token for authenticated requests +func ghClient(ctx context.Context, token string) *github.Client { + if token == "" { + return github.NewClient(nil) + } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc) +} + +// GHVersions returns current stable release and latest rc or beta pre-release +// from GitHub owner/repo repository, and any error; +// if latest pre-release version is lower than current stable release, then it +// will return current stable release for both +func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, err error) { + ghc := ghClient(ctx, ghToken) + + // walk through the paginated list of all owner/repo releases, from newest to oldest + opts := &github.ListOptions{PerPage: ghListPerPage} + for { + rls, resp, err := ghc.Repositories.ListReleases(ctx, owner, repo, opts) + if err != nil { + return "", "", err + } + for _, rl := range rls { + ver := rl.GetName() + if ver == "" { + continue + } + // check if ver version is a release (ie, 'v1.19.2') or a + // pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') channel ch + // note: github.RepositoryRelease GetPrerelease() bool would be useful for all pre-rels + ch := strings.Split(ver, "-") + if len(ch) == 1 && stable == "" { + stable = ver + } else if len(ch) > 1 && latest == "" { + if strings.HasPrefix(ch[1], "rc") || strings.HasPrefix(ch[1], "beta") { + latest = ver + } + } + if stable != "" && latest != "" { + // make sure that v.Latest >= stable + if latest < stable { + latest = stable + } + return stable, latest, nil + } + } + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + return stable, latest, nil +} diff --git a/hack/update/golang_version/update_golang_version.go b/hack/update/golang_version/update_golang_version.go new file mode 100644 index 0000000000..0e88b0debc --- /dev/null +++ b/hack/update/golang_version/update_golang_version.go @@ -0,0 +1,132 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package main + +import ( + "context" + "io/ioutil" + "net/http" + "strings" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second +) + +var ( + schema = map[string]update.Item{ + ".github/workflows/iso.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/kic_image.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/master.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".github/workflows/pr.yml": { + Replace: map[string]string{ + `go-version: '.*`: `go-version: '{{.StableVersion}}'`, + }, + }, + ".travis.yml": { + Replace: map[string]string{ + `go:\n - .*`: `go:{{printf "\n - %s" .StableVersion}}`, + `go: .*`: `go: {{.StableVersion}}`, + }, + }, + "go.mod": { + Replace: map[string]string{ + `(?m)^go .*`: `go {{.StableVersionMM}}`, + }, + }, + "hack/jenkins/common.sh": { + Replace: map[string]string{ + `sudo \.\/installers\/check_install_golang\.sh \".*\" \"\/usr\/local\"`: `sudo ./installers/check_install_golang.sh "{{.StableVersion}}" "/usr/local"`, + }, + }, + "Makefile": { + Replace: map[string]string{ + `GO_VERSION \?= .*`: `GO_VERSION ?= {{.StableVersion}}`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-golang-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update_golang_version: {stable:"{{.StableVersion}}"}` + prIssue = 9264 +) + +// Data holds stable Golang version +type Data struct { + StableVersion string `json:"stableVersion"` + StableVersionMM string `json:"stableVersionMM"` // go.mod wants go version in . format +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // get Golang stable version + stable, stableMM, err := goVersions() + if err != nil || stable == "" || stableMM == "" { + klog.Fatalf("Error getting Golang stable version: %v", err) + } + data := Data{StableVersion: stable, StableVersionMM: stableMM} + klog.Infof("Golang stable version: %s", data.StableVersion) + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} + +// goVersion returns Golang stable version +func goVersions() (stable, stableMM string, err error) { + resp, err := http.Get("https://golang.org/VERSION?m=text") + if err != nil { + return "", "", err + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", "", err + } + stable = strings.TrimPrefix(string(body), "go") + mmp := strings.SplitN(stable, ".", 3) + stableMM = strings.Join(mmp[0:2], ".") // . version + return stable, stableMM, nil +} diff --git a/hack/update/kicbase_version/update_kicbase_version.go b/hack/update/kicbase_version/update_kicbase_version.go new file mode 100644 index 0000000000..64d9432475 --- /dev/null +++ b/hack/update/kicbase_version/update_kicbase_version.go @@ -0,0 +1,148 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script promotes current KIC base image as stable, ie: + - strips current version suffix starting from '-' in pkg/drivers/kic/types.go => release version + (eg, 'v0.0.13-snapshot1' -> 'v0.0.13') + - makes sure current KIC base image exists locally, tries to pull one if not + - tags current KIC base image with the release version, and + - pushes it to all relevant container registries + +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + +Script also requires following credentials as env variables (injected by Jenkins credential provider): + @GCR (ref: https://cloud.google.com/container-registry/docs/advanced-authentication): + - GCR_USERNAME=: GCR username, eg: + = "oauth2accesstoken" if Access Token is used for GCR_TOKEN, or + = "_json_key" if JSON Key File is used for GCR_TOKEN + - GCR_TOKEN=: GCR JSON token + + @Docker (ref: https://docs.docker.com/docker-hub/access-tokens/) + - DOCKER_USERNAME=: Docker username + - DOCKER_TOKEN=: Docker personal access token or password + + @GitHub (ref: https://docs.github.com/en/free-pro-team@latest/packages/using-github-packages-with-your-projects-ecosystem/configuring-docker-for-use-with-github-packages) + - GITHUB_USERNAME=: GitHub username + - GITHUB_TOKEN=: GitHub [personal] access token +*/ + +package main + +import ( + "context" + "io/ioutil" + "path/filepath" + "regexp" + "strings" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 600 * time.Second +) + +var ( + kicFile = "pkg/drivers/kic/types.go" + versionRE = `Version = "(.*)"` + + schema = map[string]update.Item{ + kicFile: { + Replace: map[string]string{ + `Version = ".*"`: `Version = "{{.StableVersion}}"`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-kicbase-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update-kicbase-version: {"{{.StableVersion}}"}` + prIssue = 9420 +) + +// Data holds current and stable KIC Base image versions +type Data struct { + CurrentVersion string `json:"CurrentVersion"` + StableVersion string `json:"StableVersion"` +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // determine current and stable kic base image versions + current, stable, err := KICVersions() + if err != nil { + klog.Fatalf("failed getting kic base image versions: %v", err) + } + if len(current) == 0 || len(stable) == 0 { + klog.Fatalf("cannot determine kic base image versions") + } + data := Data{CurrentVersion: current, StableVersion: stable} + klog.Infof("kic base image versions: 'current' is %s and 'stable' would be %s", data.CurrentVersion, data.StableVersion) + + // prepare local kic base image + image, err := prepareImage(ctx, data) + if err != nil { + klog.Fatalf("failed preparing local kic base reference image: %v", err) + } + klog.Infof("local kic base reference image: %s", image) + + // update registries + if updated := update.CRUpdateAll(ctx, image, data.StableVersion); !updated { + klog.Fatalf("failed updating all registries") + } + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} + +// KICVersions returns current and stable kic base image versions and any error +func KICVersions() (current, stable string, err error) { + blob, err := ioutil.ReadFile(filepath.Join(update.FSRoot, kicFile)) + if err != nil { + return "", "", err + } + re := regexp.MustCompile(versionRE) + ver := re.FindSubmatch(blob) + if ver == nil { + return "", "", nil + } + current = string(ver[1]) + stable = strings.Split(current, "-")[0] + return current, stable, nil +} + +// prepareImage checks if current image exists locally, tries to pull it if not, +// tags it with release version, returns reference image url and any error +func prepareImage(ctx context.Context, data Data) (image string, err error) { + image, err = update.PullImage(ctx, data.CurrentVersion, data.StableVersion) + if err != nil { + return "", err + } + if err := update.TagImage(ctx, image, data.CurrentVersion, data.StableVersion); err != nil { + return "", err + } + return image, nil +} diff --git a/hack/update/kubernetes_version/update_kubernetes_version.go b/hack/update/kubernetes_version/update_kubernetes_version.go new file mode 100644 index 0000000000..b5bf60cef7 --- /dev/null +++ b/hack/update/kubernetes_version/update_kubernetes_version.go @@ -0,0 +1,84 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package main + +import ( + "context" + "time" + + "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" +) + +const ( + // default context timeout + cxTimeout = 300 * time.Second +) + +var ( + schema = map[string]update.Item{ + "pkg/minikube/constants/constants.go": { + Replace: map[string]string{ + `DefaultKubernetesVersion = ".*`: `DefaultKubernetesVersion = "{{.StableVersion}}"`, + `NewestKubernetesVersion = ".*`: `NewestKubernetesVersion = "{{.LatestVersion}}"`, + }, + }, + "site/content/en/docs/commands/start.md": { + Replace: map[string]string{ + `'stable' for .*,`: `'stable' for {{.StableVersion}},`, + `'latest' for .*\)`: `'latest' for {{.LatestVersion}})`, + }, + }, + } + + // pull request data + prBranchPrefix = "update-kubernetes-version_" // will be appended with first 7 characters of the PR commit SHA + prTitle = `update_kubernetes_version: {stable:"{{.StableVersion}}", latest:"{{.LatestVersion}}"}` + prIssue = 4392 +) + +// Data holds stable and latest Kubernetes versions +type Data struct { + StableVersion string `json:"StableVersion"` + LatestVersion string `json:"LatestVersion"` +} + +func main() { + // set a context with defined timeout + ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) + defer cancel() + + // get Kubernetes versions from GitHub Releases + stable, latest, err := update.GHVersions(ctx, "kubernetes", "kubernetes") + if err != nil || stable == "" || latest == "" { + klog.Fatalf("Error getting Kubernetes versions: %v", err) + } + data := Data{StableVersion: stable, LatestVersion: latest} + klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.StableVersion, data.LatestVersion) + + update.Apply(ctx, schema, data, prBranchPrefix, prTitle, prIssue) +} diff --git a/hack/update/registry.go b/hack/update/registry.go new file mode 100644 index 0000000000..5ce7bb1a93 --- /dev/null +++ b/hack/update/registry.go @@ -0,0 +1,134 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package update + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "time" + + "k8s.io/klog/v2" +) + +var ( + // keep list of registries in sync with those in "pkg/drivers/kic/types.go" + registries = []registry{ + { + name: "Google Cloud Container Registry", + image: "gcr.io/k8s-minikube/kicbase", + username: os.Getenv("GCR_USERNAME"), + password: os.Getenv("GCR_TOKEN"), + }, + { + name: "Docker Hub Container Registry", + image: "docker.io/kicbase/stable", + username: os.Getenv("DOCKER_USERNAME"), + password: os.Getenv("DOCKER_TOKEN"), + }, + { + name: "GitHub Packages Registry", + image: "docker.pkg.github.com/kubernetes/minikube/kicbase", + username: os.Getenv("GITHUB_USERNAME"), + password: os.Getenv("GITHUB_TOKEN"), + }, + } +) + +// container registry name, image path, credentials, and updated flag +type registry struct { + name string + image string + username string + password string +} + +// crUpdate tags image with version, pushes it to container registry, and returns any error +func crUpdate(ctx context.Context, reg registry, image, version string) error { + login := exec.CommandContext(ctx, "docker", "login", "--username", reg.username, "--password-stdin", reg.image) + if err := RunWithRetryNotify(ctx, login, strings.NewReader(reg.password), 1*time.Minute, 10); err != nil { + return fmt.Errorf("failed logging in to %s: %w", reg.name, err) + } + klog.Infof("successfully logged in to %s", reg.name) + + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+version, reg.image+":"+version) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Minute, 10); err != nil { + return fmt.Errorf("failed tagging %s for %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("successfully tagged %s for %s", reg.image+":"+version, reg.name) + + push := exec.CommandContext(ctx, "docker", "push", reg.image+":"+version) + if err := RunWithRetryNotify(ctx, push, nil, 2*time.Minute, 10); err != nil { + return fmt.Errorf("failed pushing %s to %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("successfully pushed %s to %s", reg.image+":"+version, reg.name) + + return nil +} + +// CRUpdateAll calls crUpdate for each available registry, and returns if at least one got updated +func CRUpdateAll(ctx context.Context, image, version string) (updated bool) { + for _, reg := range registries { + if err := crUpdate(ctx, reg, image, version); err != nil { + klog.Errorf("failed updating %s", reg.name) + continue + } + klog.Infof("successfully updated %s", reg.name) + updated = true + } + return updated +} + +// PullImage checks if current image exists locally, tries to pull it if not, and +// returns reference image url and any error +func PullImage(ctx context.Context, current, release string) (image string, err error) { + // check if image exists locally + for _, reg := range registries { + inspect := exec.CommandContext(ctx, "docker", "inspect", reg.image+":"+current, "--format", "{{.Id}}") + if err := RunWithRetryNotify(ctx, inspect, nil, 1*time.Second, 10); err != nil { + continue + } + image = reg.image + break + } + if image == "" { + // try to pull image locally + for _, reg := range registries { + pull := exec.CommandContext(ctx, "docker", "pull", reg.image+":"+current) + if err := RunWithRetryNotify(ctx, pull, nil, 2*time.Minute, 10); err != nil { + continue + } + image = reg.image + break + } + } + if image == "" { + return "", fmt.Errorf("cannot find current image version tag %s locally nor in any registry", current) + } + return image, nil +} + +// TagImage tags local image:current with stable version, and returns any error +func TagImage(ctx context.Context, image, current, stable string) error { + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+current, image+":"+stable) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Second, 10); err != nil { + return err + } + return nil +} diff --git a/hack/update/update.go b/hack/update/update.go new file mode 100644 index 0000000000..385964bfa3 --- /dev/null +++ b/hack/update/update.go @@ -0,0 +1,212 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +Script expects the following env variables: + - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: + - "fs" - update only local filesystem repo files [default] + - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) + - "all" - update local and remote repo files and create PR (if one does not exist already) + - GITHUB_TOKEN=: GitHub [personal] access token + - note: GITHUB_TOKEN is required if UPDATE_TARGET is "gh" or "all" +*/ + +package update + +import ( + "bytes" + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "text/template" + "time" + + "k8s.io/klog/v2" + + "github.com/cenkalti/backoff/v4" +) + +const ( + // FSRoot is relative (to scripts in subfolders) root folder of local filesystem repo to update + FSRoot = "../../../" +) + +var ( + target = os.Getenv("UPDATE_TARGET") +) + +// init klog and check general requirements +func init() { + // write log statements to stderr instead of to files + if err := flag.Set("logtostderr", "true"); err != nil { + fmt.Printf("Error setting 'logtostderr' klog flag: %v", err) + } + flag.Parse() + defer klog.Flush() + + if target == "" { + target = "fs" + } else if target != "fs" && target != "gh" && target != "all" { + klog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) + } else if (target == "gh" || target == "all") && ghToken == "" { + klog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") + } +} + +// Item defines Content where all occurrences of each Replace map key, corresponding to +// GitHub TreeEntry.Path and/or local filesystem repo file path (prefixed with FSRoot), +// would be swapped with its respective actual map value (having placeholders replaced with data), +// creating a concrete update plan. +// Replace map keys can use RegExp and map values can use Golang Text Template +type Item struct { + Content []byte `json:"-"` + Replace map[string]string `json:"replace"` +} + +// apply updates Item Content by replacing all occurrences of Replace map's keys +// with their actual map values (with placeholders replaced with data)) +func (i *Item) apply(data interface{}) (changed bool, err error) { + if i.Content == nil || i.Replace == nil { + return false, fmt.Errorf("want something, got nothing to update") + } + org := string(i.Content) + str := org + for src, dst := range i.Replace { + tmpl := template.Must(template.New("").Parse(dst)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + return false, err + } + re := regexp.MustCompile(src) + str = re.ReplaceAllString(str, buf.String()) + } + i.Content = []byte(str) + + return str != org, nil +} + +// Apply applies concrete update plan (schema + data) to GitHub or local filesystem repo +func Apply(ctx context.Context, schema map[string]Item, data interface{}, prBranchPrefix, prTitle string, prIssue int) { + plan, err := GetPlan(schema, data) + if err != nil { + klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + } + klog.Infof("The Plan:\n%s", plan) + + if target == "fs" || target == "all" { + changed, err := fsUpdate(FSRoot, schema, data) + if err != nil { + klog.Errorf("Error updating local repo: %v", err) + } else if !changed { + klog.Infof("Local repo update skipped: nothing changed") + } else { + klog.Infof("Local repo updated") + } + } + + if target == "gh" || target == "all" { + // update prTitle replacing template placeholders with actual data values + tmpl := template.Must(template.New("prTitle").Parse(prTitle)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + klog.Fatalf("Error parsing PR Title: %v", err) + } + prTitle = buf.String() + + // check if PR already exists + prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) + if err != nil { + klog.Errorf("Error checking if PR already exists: %v", err) + } else if prURL != "" { + klog.Infof("PR create skipped: already exists (%s)", prURL) + } else { + // create PR + pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, schema, data) + if err != nil { + klog.Fatalf("Error creating PR: %v", err) + } else if pr == nil { + klog.Infof("PR create skipped: nothing changed") + } else { + klog.Infof("PR created: %s", *pr.HTMLURL) + } + } + } +} + +// GetPlan returns concrete plan replacing placeholders in schema with actual data values, +// returns JSON-formatted representation of the plan and any error +func GetPlan(schema map[string]Item, data interface{}) (prettyprint string, err error) { + for _, item := range schema { + for src, dst := range item.Replace { + tmpl := template.Must(template.New("").Parse(dst)) + buf := new(bytes.Buffer) + if err := tmpl.Execute(buf, data); err != nil { + return fmt.Sprintf("%+v", schema), err + } + item.Replace[src] = buf.String() + } + } + str, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return fmt.Sprintf("%+v", schema), err + } + return string(str), nil +} + +// RunWithRetryNotify runs command cmd with stdin using exponential backoff for maxTime duration +// up to maxRetries (negative values will make it ignored), +// notifies about any intermediary errors and return any final error. +// similar to pkg/util/retry/retry.go:Expo(), just for commands with params and also with context +func RunWithRetryNotify(ctx context.Context, cmd *exec.Cmd, stdin io.Reader, maxTime time.Duration, maxRetries uint64) error { + be := backoff.NewExponentialBackOff() + be.Multiplier = 2 + be.MaxElapsedTime = maxTime + bm := backoff.WithMaxRetries(be, maxRetries) + bc := backoff.WithContext(bm, ctx) + + notify := func(err error, wait time.Duration) { + klog.Errorf("Temporary error running '%s' (will retry in %s): %v", cmd.String(), wait, err) + } + if err := backoff.RetryNotify(func() error { + cmd.Stdin = stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + time.Sleep(be.NextBackOff().Round(1 * time.Second)) + return fmt.Errorf("%w: %s", err, stderr.String()) + } + return nil + }, bc, notify); err != nil { + return err + } + return nil +} + +// Run runs command cmd with stdin +func Run(cmd *exec.Cmd, stdin io.Reader) error { + cmd.Stdin = stdin + var out bytes.Buffer + cmd.Stderr = &out + if err := cmd.Run(); err != nil { + return fmt.Errorf("%w: %s", err, out.String()) + } + return nil +}