diff --git a/go.mod b/go.mod index 8292131c30..cb5b07c888 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,6 @@ require ( github.com/blang/semver v3.5.0+incompatible github.com/c4milo/gotoolkit v0.0.0-20170318115440-bcc06269efa9 // indirect github.com/cenkalti/backoff v2.2.1+incompatible - github.com/cenkalti/backoff/v4 v4.1.0 github.com/cheggaaa/pb/v3 v3.0.1 github.com/cloudevents/sdk-go/v2 v2.1.0 github.com/cloudfoundry-attic/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 @@ -76,7 +75,6 @@ require ( golang.org/x/build v0.0.0-20190927031335-2835ba2e683f golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 - golang.org/x/mod v0.3.0 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a golang.org/x/sys v0.0.0-20200523222454-059865788121 diff --git a/go.sum b/go.sum index ca950495ac..2e3dad0cf6 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,6 @@ github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e/go.mod h1:oD github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/cenkalti/backoff/v4 v4.1.0 h1:c8LkOFQTzuO0WBM/ae5HdGQuZPfPxp7lqBRwQRm4fSc= -github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/prettybench v0.0.0-20150116022406-03b8cfe5406c/go.mod h1:Xe6ZsFhtM8HrDku0pxJ3/Lr51rwykrzgFwpmTzleatY= @@ -604,6 +602,7 @@ github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6K github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-getter v1.4.2/go.mod h1:3Ao9Hol5VJsmwJV5BF1GUrONbaOUmA+m1Nj2+0LuMAY= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= diff --git a/hack/kubernetes_version/update_kubernetes_version.go b/hack/kubernetes_version/update_kubernetes_version.go deleted file mode 100644 index 8d68b61324..0000000000 --- a/hack/kubernetes_version/update_kubernetes_version.go +++ /dev/null @@ -1,458 +0,0 @@ -/* -Copyright 2020 The Kubernetes Authors All rights reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -The script expects the following env variables: - - UPDATE_TARGET=: optional - if unset/absent, default option is "fs"; valid options are: - - "fs" - update only local filesystem repo files [default] - - "gh" - update only remote GitHub repo files and create PR (if one does not exist already) - - "all" - update local and remote repo files and create PR (if one does not exist already) - - GITHUB_TOKEN=: The Github API access token. Injected by the Jenkins credential provider. - - note: GITHUB_TOKEN is needed only if UPDATE_TARGET is "gh" or "all" -*/ - -package main - -import ( - "bytes" - "context" - "encoding/json" - "flag" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "regexp" - "strings" - "text/template" - "time" - - "golang.org/x/mod/semver" - "golang.org/x/oauth2" - - "github.com/google/go-github/v32/github" - - "k8s.io/klog/v2" -) - -const ( - // default context timeout - cxTimeout = 300 * time.Second - - // use max value (100) for PerPage to avoid hitting the rate limits (60 per hour, 10 per minute) - // see https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting - ghListOptionsPerPage = 100 -) - -var ( - // root directory of the local filesystem repo to update - fsRoot = "../../" - - // map key corresponds to GitHub TreeEntry.Path and local repo file path (prefixed with fsRoot) - plan = map[string]Patch{ - "pkg/minikube/constants/constants.go": { - Replace: map[string]string{ - `DefaultKubernetesVersion = \".*`: `DefaultKubernetesVersion = "{{.K8sStableVersion}}"`, - `NewestKubernetesVersion = \".*`: `NewestKubernetesVersion = "{{.K8sLatestVersion}}"`, - }, - }, - "site/content/en/docs/commands/start.md": { - Replace: map[string]string{ - `'stable' for .*,`: `'stable' for {{.K8sStableVersion}},`, - `'latest' for .*\)`: `'latest' for {{.K8sLatestVersion}})`, - }, - }, - } - - target = os.Getenv("UPDATE_TARGET") - - // GitHub repo data - ghToken = os.Getenv("GITHUB_TOKEN") - ghOwner = "kubernetes" - ghRepo = "minikube" - ghBase = "master" // could be "main" in the future? - - // PR data - prBranchPrefix = "update-kubernetes-version_" // will be appended with first 7 characters of the PR commit SHA - prTitle = `update_kubernetes_version: {stable:"{{.K8sStableVersion}}", latest:"{{.K8sLatestVersion}}"}` - prIssue = 4392 - prSearchLimit = 100 // limit the number of previous PRs searched for same prTitle to be <= N * ghListOptionsPerPage -) - -// Data holds respective stable (release) and latest (pre-release) Kubernetes versions -type Data struct { - K8sStableVersion string `json:"k8sStableVersion"` - K8sLatestVersion string `json:"k8sLatestVersion"` -} - -// Patch defines content where all occurrences of each replace map key should be swapped with its -// respective value. Replace map keys can use RegExp and values can use Golang Text Template -type Patch struct { - Content []byte `json:"-"` - Replace map[string]string `json:"replace"` -} - -// apply patch to content by replacing all occurrences of map's keys with their respective values -func (p *Patch) apply(data interface{}) (changed bool, err error) { - if p.Content == nil || p.Replace == nil { - return false, fmt.Errorf("nothing to patch") - } - org := string(p.Content) - str := org - for src, dst := range p.Replace { - re := regexp.MustCompile(src) - tmpl := template.Must(template.New("").Parse(dst)) - buf := new(bytes.Buffer) - if err := tmpl.Execute(buf, data); err != nil { - return false, err - } - str = re.ReplaceAllString(str, buf.String()) - } - p.Content = []byte(str) - - return str != org, nil -} - -func main() { - klog.InitFlags(nil) - // write log statements to stderr instead of to files - if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' klog flag: %v\n", err) - } - flag.Parse() - defer klog.Flush() - - if target == "" { - target = "fs" - } else if target != "fs" && target != "gh" && target != "all" { - klog.Fatalf("Invalid UPDATE_TARGET option: '%s'; Valid options are: unset/absent (defaults to 'fs'), 'fs', 'gh', or 'all'", target) - } else if (target == "gh" || target == "all") && ghToken == "" { - klog.Fatalf("GITHUB_TOKEN is required if UPDATE_TARGET is 'gh' or 'all'") - } - - // set a context with defined timeout - ctx, cancel := context.WithTimeout(context.Background(), cxTimeout) - defer cancel() - - // get Kubernetes versions from GitHub Releases - stable, latest, err := ghReleases(ctx, "kubernetes", "kubernetes", ghToken) - if err != nil || stable == "" || latest == "" { - klog.Fatalf("Error getting Kubernetes versions: %v", err) - } - data := Data{K8sStableVersion: stable, K8sLatestVersion: latest} - klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.K8sStableVersion, data.K8sLatestVersion) - - klog.Infof("The Plan:\n%s", thePlan(plan, data)) - - if target == "fs" || target == "all" { - changed, err := fsUpdate(fsRoot, plan, data) - if err != nil { - klog.Errorf("Error updating local repo: %v", err) - } else if !changed { - klog.Infof("Local repo update skipped: nothing changed") - } else { - klog.Infof("Local repo updated") - } - } - - if target == "gh" || target == "all" { - // update prTitle replacing template placeholders with concrete data values - tmpl := template.Must(template.New("prTitle").Parse(prTitle)) - buf := new(bytes.Buffer) - if err := tmpl.Execute(buf, data); err != nil { - klog.Fatalf("Error parsing PR Title: %v", err) - } - prTitle = buf.String() - - // check if PR already exists - prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) - if err != nil { - klog.Errorf("Error checking if PR already exists: %v", err) - } else if prURL != "" { - klog.Infof("PR create skipped: already exists (%s)", prURL) - } else { - // create PR - pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, plan, data) - if err != nil { - klog.Fatalf("Error creating PR: %v", err) - } else if pr == nil { - klog.Infof("PR create skipped: nothing changed") - } else { - klog.Infof("PR created: %s", *pr.HTMLURL) - } - } - } -} - -// fsUpdate updates local filesystem repo files according to the given plan and data, -// returns if the update actually changed anything, and any error occurred -func fsUpdate(fsRoot string, plan map[string]Patch, data Data) (changed bool, err error) { - for path, p := range plan { - path = filepath.Join(fsRoot, path) - blob, err := ioutil.ReadFile(path) - if err != nil { - return false, err - } - info, err := os.Stat(path) - if err != nil { - return false, err - } - mode := info.Mode() - - p.Content = blob - chg, err := p.apply(data) - if err != nil { - return false, err - } - if chg { - changed = true - } - if err := ioutil.WriteFile(path, p.Content, mode); err != nil { - return false, err - } - } - return changed, nil -} - -// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head -// commit fork, as defined by the plan and data, and also returns any error occurred -// PR branch will be named by the branch, sufixed by '_' and first 7 characters of fork commit SHA -// PR itself will be named by the title and will reference the issue -func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, issue int, token string, plan map[string]Patch, data Data) (*github.PullRequest, error) { - ghc := ghClient(ctx, token) - - // get base branch - baseBranch, _, err := ghc.Repositories.GetBranch(ctx, owner, repo, base) - if err != nil { - return nil, fmt.Errorf("error getting base branch: %w", err) - } - - // get base commit - baseCommit, _, err := ghc.Repositories.GetCommit(ctx, owner, repo, *baseBranch.Commit.SHA) - if err != nil { - return nil, fmt.Errorf("error getting base commit: %w", err) - } - - // get base tree - baseTree, _, err := ghc.Git.GetTree(ctx, owner, repo, baseCommit.GetSHA(), true) - if err != nil { - return nil, fmt.Errorf("error getting base tree: %w", err) - } - - // update files - changes, err := ghUpdate(ctx, owner, repo, baseTree, token, plan, data) - if err != nil { - return nil, fmt.Errorf("error updating files: %w", err) - } - if changes == nil { - return nil, nil - } - - // create fork - fork, resp, err := ghc.Repositories.CreateFork(ctx, owner, repo, nil) - // https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork - // This method might return an *AcceptedError and a status code of 202. This is because this is - // the status that GitHub returns to signify that it is now computing creating the fork in a - // background task. In this event, the Repository value will be returned, which includes the - // details about the pending fork. A follow up request, after a delay of a second or so, should - // result in a successful request. - if resp.StatusCode == 202 { // *AcceptedError - time.Sleep(time.Second * 5) - } else if err != nil { - return nil, fmt.Errorf("error creating fork: %w", err) - } - - // create fork tree from base and changed files - forkTree, _, err := ghc.Git.CreateTree(ctx, *fork.Owner.Login, *fork.Name, *baseTree.SHA, changes) - if err != nil { - return nil, fmt.Errorf("error creating fork tree: %w", err) - } - - // create fork commit - forkCommit, _, err := ghc.Git.CreateCommit(ctx, *fork.Owner.Login, *fork.Name, &github.Commit{ - Message: github.String(title), - Tree: &github.Tree{SHA: forkTree.SHA}, - Parents: []*github.Commit{{SHA: baseCommit.SHA}}, - }) - if err != nil { - return nil, fmt.Errorf("error creating fork commit: %w", err) - } - klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) - - // create PR branch - prBranch := branch + forkCommit.GetSHA()[:7] - prRef, _, err := ghc.Git.CreateRef(ctx, *fork.Owner.Login, *fork.Name, &github.Reference{ - Ref: github.String("refs/heads/" + prBranch), - Object: &github.GitObject{ - Type: github.String("commit"), - SHA: forkCommit.SHA, - }, - }) - if err != nil { - return nil, fmt.Errorf("error creating PR branch: %w", err) - } - klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) - - // create PR - modifiable := true - pr, _, err := ghc.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{ - Title: github.String(title), - Head: github.String(*fork.Owner.Login + ":" + prBranch), - Base: github.String(base), - Body: github.String(fmt.Sprintf("fixes #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, thePlan(plan, data))), - MaintainerCanModify: &modifiable, - }) - if err != nil { - return nil, fmt.Errorf("error creating pull request: %w", err) - } - return pr, nil -} - -// ghUpdate updates remote GitHub owner/repo tree according to the given token, plan and data, -// returns resulting changes, and any error occurred -func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, plan map[string]Patch, data Data) (changes []*github.TreeEntry, err error) { - ghc := ghClient(ctx, token) - - // load each plan's path content and update it creating new GitHub TreeEntries - cnt := len(plan) // expected number of files to change - for _, org := range tree.Entries { - if *org.Type == "blob" { - if patch, match := plan[*org.Path]; match { - blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA) - if err != nil { - return nil, fmt.Errorf("error getting file: %w", err) - } - patch.Content = blob - changed, err := patch.apply(data) - if err != nil { - return nil, fmt.Errorf("error patching file: %w", err) - } - if changed { - // add github.TreeEntry that will replace original path content with patched one - changes = append(changes, &github.TreeEntry{ - Path: org.Path, - Mode: org.Mode, - Type: org.Type, - Content: github.String(string(patch.Content)), - }) - } - if cnt--; cnt == 0 { - break - } - } - } - } - if cnt != 0 { - return nil, fmt.Errorf("error finding all the files (%d missing) - check the Plan: %w", cnt, err) - } - return changes, nil -} - -// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred -func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url string, err error) { - ghc := ghClient(ctx, token) - - // walk through the paginated list of all pull requests, from latest to older releases - opts := &github.PullRequestListOptions{State: "all", Base: base, ListOptions: github.ListOptions{PerPage: ghListOptionsPerPage}} - for (opts.Page+1)*ghListOptionsPerPage <= prSearchLimit { - prs, resp, err := ghc.PullRequests.List(ctx, owner, repo, opts) - if err != nil { - return "", err - } - for _, pr := range prs { - if pr.GetTitle() == title { - return pr.GetHTMLURL(), nil - } - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - return "", nil -} - -// ghReleases returns current stable release and latest rc or beta pre-release -// from GitHub owner/repo repository, and any error; -// if latest pre-release version is lower than current stable release, then it -// will return current stable release for both -func ghReleases(ctx context.Context, owner, repo, token string) (stable, latest string, err error) { - ghc := ghClient(ctx, token) - - // walk through the paginated list of all owner/repo releases, from newest to oldest - opts := &github.ListOptions{PerPage: ghListOptionsPerPage} - for { - rls, resp, err := ghc.Repositories.ListReleases(ctx, owner, repo, opts) - if err != nil { - return "", "", err - } - for _, rl := range rls { - ver := rl.GetName() - if !semver.IsValid(ver) { - continue - } - // check if ver version is release (ie, 'v1.19.2') or pre-release (ie, 'v1.19.3-rc.0' or 'v1.19.0-beta.2') - prerls := semver.Prerelease(ver) - if prerls == "" { - stable = semver.Max(ver, stable) - } else if strings.HasPrefix(prerls, "-rc") || strings.HasPrefix(prerls, "-beta") { - latest = semver.Max(ver, latest) - } - // make sure that latest >= stable - if semver.Compare(latest, stable) == -1 { - latest = stable - } - } - if resp.NextPage == 0 { - break - } - opts.Page = resp.NextPage - } - return stable, latest, nil -} - -// ghClient returns GitHub Client with a given context and optional token for authenticated requests -func ghClient(ctx context.Context, token string) *github.Client { - if token == "" { - return github.NewClient(nil) - } - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(ctx, ts) - return github.NewClient(tc) -} - -// thePlan parses and returns updated plan replacing template placeholders with concrete data values -func thePlan(plan map[string]Patch, data Data) (prettyprint string) { - for _, p := range plan { - for src, dst := range p.Replace { - tmpl := template.Must(template.New("").Parse(dst)) - buf := new(bytes.Buffer) - if err := tmpl.Execute(buf, data); err != nil { - klog.Fatalf("Error parsing the Plan: %v", err) - return fmt.Sprintf("%+v", plan) - } - p.Replace[src] = buf.String() - } - } - str, err := json.MarshalIndent(plan, "", " ") - if err != nil { - klog.Fatalf("Error parsing the Plan: %v", err) - return fmt.Sprintf("%+v", plan) - } - return string(str) -} diff --git a/hack/update/filesystem.go b/hack/update/filesystem.go index 037c0ffcd6..864063f5f1 100644 --- a/hack/update/filesystem.go +++ b/hack/update/filesystem.go @@ -22,8 +22,8 @@ import ( "path/filepath" ) -// fsUpdate updates local filesystem repo files according to the given schema and data, -// returns if the update actually changed anything, and any error occurred +// fsUpdate updates local filesystem repo files according to the given schema and data. +// Returns if the update actually changed anything, and any error occurred. func fsUpdate(fsRoot string, schema map[string]Item, data interface{}) (changed bool, err error) { for path, item := range schema { path = filepath.Join(fsRoot, path) diff --git a/hack/update/github.go b/hack/update/github.go index 661cbc0fa5..cefe99b1fb 100644 --- a/hack/update/github.go +++ b/hack/update/github.go @@ -31,11 +31,11 @@ import ( ) const ( - // ghListPerPage uses max value (100) for PerPage to avoid hitting the rate limits + // ghListPerPage uses max value (100) for PerPage to avoid hitting the rate limits. // (ref: https://godoc.org/github.com/google/go-github/github#hdr-Rate_Limiting) ghListPerPage = 100 - // ghSearchLimit limits the number of searched items to be <= N * ListPerPage + // ghSearchLimit limits the number of searched items to be <= N * ghListPerPage. ghSearchLimit = 100 ) @@ -44,38 +44,38 @@ var ( ghToken = os.Getenv("GITHUB_TOKEN") ghOwner = "kubernetes" ghRepo = "minikube" - ghBase = "master" // could be "main" in the future? + ghBase = "master" // could be "main" in the near future? ) -// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head -// commit fork, as defined by the schema and data, and also returns any error occurred -// PR branch will be named by the branch, sufixed by '_' and first 7 characters of fork commit SHA -// PR itself will be named by the title and will reference the issue +// ghCreatePR returns PR created in the GitHub owner/repo, applying the changes to the base head commit fork, as defined by the schema and data. +// Returns any error occurred. +// PR branch will be named by the branch, sufixed by '_' and first 7 characters of the fork commit SHA. +// PR itself will be named by the title and will reference the issue. func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, issue int, token string, schema map[string]Item, data interface{}) (*github.PullRequest, error) { ghc := ghClient(ctx, token) // get base branch baseBranch, _, err := ghc.Repositories.GetBranch(ctx, owner, repo, base) if err != nil { - return nil, fmt.Errorf("error getting base branch: %w", err) + return nil, fmt.Errorf("unable to get base branch: %w", err) } // get base commit baseCommit, _, err := ghc.Repositories.GetCommit(ctx, owner, repo, *baseBranch.Commit.SHA) if err != nil { - return nil, fmt.Errorf("error getting base commit: %w", err) + return nil, fmt.Errorf("unable to get base commit: %w", err) } // get base tree baseTree, _, err := ghc.Git.GetTree(ctx, owner, repo, baseCommit.GetSHA(), true) if err != nil { - return nil, fmt.Errorf("error getting base tree: %w", err) + return nil, fmt.Errorf("unable to get base tree: %w", err) } // update files changes, err := ghUpdate(ctx, owner, repo, baseTree, token, schema, data) if err != nil { - return nil, fmt.Errorf("error updating files: %w", err) + return nil, fmt.Errorf("unable to update files: %w", err) } if changes == nil { return nil, nil @@ -83,22 +83,21 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is // create fork fork, resp, err := ghc.Repositories.CreateFork(ctx, owner, repo, nil) - // https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork - // This method might return an *AcceptedError and a status code of 202. This is because this is - // the status that GitHub returns to signify that it is now computing creating the fork in a - // background task. In this event, the Repository value will be returned, which includes the - // details about the pending fork. A follow up request, after a delay of a second or so, should - // result in a successful request. + // "This method might return an *AcceptedError and a status code of 202. + // This is because this is the status that GitHub returns to signify that it is now computing creating the fork in a background task. + // In this event, the Repository value will be returned, which includes the details about the pending fork. + // A follow up request, after a delay of a second or so, should result in a successful request." + // (ref: https://pkg.go.dev/github.com/google/go-github/v32@v32.1.0/github#RepositoriesService.CreateFork) if resp.StatusCode == 202 { // *AcceptedError time.Sleep(time.Second * 5) } else if err != nil { - return nil, fmt.Errorf("error creating fork: %w", err) + return nil, fmt.Errorf("unable to create fork: %w", err) } // create fork tree from base and changed files forkTree, _, err := ghc.Git.CreateTree(ctx, *fork.Owner.Login, *fork.Name, *baseTree.SHA, changes) if err != nil { - return nil, fmt.Errorf("error creating fork tree: %w", err) + return nil, fmt.Errorf("unable to create fork tree: %w", err) } // create fork commit @@ -108,9 +107,9 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is Parents: []*github.Commit{{SHA: baseCommit.SHA}}, }) if err != nil { - return nil, fmt.Errorf("error creating fork commit: %w", err) + return nil, fmt.Errorf("unable to create fork commit: %w", err) } - klog.Infof("PR commit '%s' created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) + klog.Infof("PR commit '%s' successfully created: %s", forkCommit.GetSHA(), forkCommit.GetHTMLURL()) // create PR branch prBranch := branch + forkCommit.GetSHA()[:7] @@ -122,74 +121,34 @@ func ghCreatePR(ctx context.Context, owner, repo, base, branch, title string, is }, }) if err != nil { - return nil, fmt.Errorf("error creating PR branch: %w", err) + return nil, fmt.Errorf("unable to create PR branch: %w", err) } - klog.Infof("PR branch '%s' created: %s", prBranch, prRef.GetURL()) + klog.Infof("PR branch '%s' successfully created: %s", prBranch, prRef.GetURL()) // create PR plan, err := GetPlan(schema, data) if err != nil { - klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + klog.Fatalf("Unable to parse schema: %v\n%s", err, plan) } modifiable := true pr, _, err := ghc.PullRequests.Create(ctx, owner, repo, &github.NewPullRequest{ Title: github.String(title), Head: github.String(*fork.Owner.Login + ":" + prBranch), Base: github.String(base), - Body: github.String(fmt.Sprintf("fixes #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, plan)), + Body: github.String(fmt.Sprintf("fixes: #%d\n\nAutomatically created PR to update repo according to the Plan:\n\n```\n%s\n```", issue, plan)), MaintainerCanModify: &modifiable, }) if err != nil { - return nil, fmt.Errorf("error creating pull request: %w", err) + return nil, fmt.Errorf("unable to create PR: %w", err) } return pr, nil } -// ghUpdate updates remote GitHub owner/repo tree according to the given token, schema and data, -// returns resulting changes, and any error occurred -func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, schema map[string]Item, data interface{}) (changes []*github.TreeEntry, err error) { - ghc := ghClient(ctx, token) - - // load each schema item content and update it creating new GitHub TreeEntries - cnt := len(schema) // expected number of files to change - for _, org := range tree.Entries { - if *org.Type == "blob" { - if item, match := schema[*org.Path]; match { - blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA) - if err != nil { - return nil, fmt.Errorf("error getting file: %w", err) - } - item.Content = blob - changed, err := item.apply(data) - if err != nil { - return nil, fmt.Errorf("error updating file: %w", err) - } - if changed { - // add github.TreeEntry that will replace original path content with updated one - changes = append(changes, &github.TreeEntry{ - Path: org.Path, - Mode: org.Mode, - Type: org.Type, - Content: github.String(string(item.Content)), - }) - } - if cnt--; cnt == 0 { - break - } - } - } - } - if cnt != 0 { - return nil, fmt.Errorf("error finding all the files (%d missing) - check the Plan: %w", cnt, err) - } - return changes, nil -} - -// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred +// ghFindPR returns URL of the PR if found in the given GitHub ower/repo base and any error occurred. func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url string, err error) { ghc := ghClient(ctx, token) - // walk through the paginated list of all pull requests, from latest to older releases + // walk through the paginated list of up to ghSearchLimit newest pull requests opts := &github.PullRequestListOptions{State: "all", Base: base, ListOptions: github.ListOptions{PerPage: ghListPerPage}} for (opts.Page+1)*ghListPerPage <= ghSearchLimit { prs, resp, err := ghc.PullRequests.List(ctx, owner, repo, opts) @@ -209,28 +168,54 @@ func ghFindPR(ctx context.Context, title, owner, repo, base, token string) (url return "", nil } -// ghClient returns GitHub Client with a given context and optional token for authenticated requests -func ghClient(ctx context.Context, token string) *github.Client { - if token == "" { - return github.NewClient(nil) +// ghUpdate updates remote GitHub owner/repo tree according to the given token, schema and data. +// Returns resulting changes, and any error occurred. +func ghUpdate(ctx context.Context, owner, repo string, tree *github.Tree, token string, schema map[string]Item, data interface{}) (changes []*github.TreeEntry, err error) { + ghc := ghClient(ctx, token) + + // load each schema item content and update it creating new GitHub TreeEntries + cnt := len(schema) // expected number of files to change + for _, org := range tree.Entries { + if *org.Type == "blob" { + if item, match := schema[*org.Path]; match { + blob, _, err := ghc.Git.GetBlobRaw(ctx, owner, repo, *org.SHA) + if err != nil { + return nil, fmt.Errorf("unable to get file: %w", err) + } + item.Content = blob + changed, err := item.apply(data) + if err != nil { + return nil, fmt.Errorf("unable to update file: %w", err) + } + if changed { + // add github.TreeEntry that will replace original path content with the updated one + changes = append(changes, &github.TreeEntry{ + Path: org.Path, + Mode: org.Mode, + Type: org.Type, + Content: github.String(string(item.Content)), + }) + } + if cnt--; cnt == 0 { + break + } + } + } } - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(ctx, ts) - return github.NewClient(tc) + if cnt != 0 { + return nil, fmt.Errorf("unable to find all the files (%d missing) - check the Plan: %w", cnt, err) + } + return changes, nil } -// GHVersions returns greatest current stable release and greatest latest rc or beta pre-release -// from GitHub owner/repo repository, and any error; -// if latest pre-release version is lower than current stable release, then it -// will return current stable release for both -func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, err error) { +// GHReleases returns greatest current stable release and greatest latest rc or beta pre-release from GitHub owner/repo repository, and any error occurred. +// If latest pre-release version is lower than the current stable release, then it will return current stable release for both. +func GHReleases(ctx context.Context, owner, repo string) (stable, latest string, err error) { ghc := ghClient(ctx, ghToken) - // walk through the paginated list of all owner/repo releases, from newest to oldest + // walk through the paginated list of up to ghSearchLimit newest releases opts := &github.ListOptions{PerPage: ghListPerPage} - for { + for (opts.Page+1)*ghListPerPage <= ghSearchLimit { rls, resp, err := ghc.Repositories.ListReleases(ctx, owner, repo, opts) if err != nil { return "", "", err @@ -259,3 +244,15 @@ func GHVersions(ctx context.Context, owner, repo string) (stable, latest string, } return stable, latest, nil } + +// ghClient returns GitHub Client with a given context and optional token for authenticated requests. +func ghClient(ctx context.Context, token string) *github.Client { + if token == "" { + return github.NewClient(nil) + } + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: token}, + ) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc) +} diff --git a/hack/update/kubernetes_version/update_kubernetes_version.go b/hack/update/kubernetes_version/update_kubernetes_version.go index b5bf60cef7..92f910b292 100644 --- a/hack/update/kubernetes_version/update_kubernetes_version.go +++ b/hack/update/kubernetes_version/update_kubernetes_version.go @@ -31,6 +31,7 @@ import ( "time" "k8s.io/klog/v2" + "k8s.io/minikube/hack/update" ) @@ -55,13 +56,13 @@ var ( }, } - // pull request data + // PR data prBranchPrefix = "update-kubernetes-version_" // will be appended with first 7 characters of the PR commit SHA - prTitle = `update_kubernetes_version: {stable:"{{.StableVersion}}", latest:"{{.LatestVersion}}"}` + prTitle = `update_kubernetes_version: {stable: "{{.StableVersion}}", latest: "{{.LatestVersion}}"}` prIssue = 4392 ) -// Data holds stable and latest Kubernetes versions +// Data holds greatest current stable release and greatest latest rc or beta pre-release Kubernetes versions type Data struct { StableVersion string `json:"StableVersion"` LatestVersion string `json:"LatestVersion"` @@ -73,9 +74,9 @@ func main() { defer cancel() // get Kubernetes versions from GitHub Releases - stable, latest, err := update.GHVersions(ctx, "kubernetes", "kubernetes") + stable, latest, err := update.GHReleases(ctx, "kubernetes", "kubernetes") if err != nil || stable == "" || latest == "" { - klog.Fatalf("Error getting Kubernetes versions: %v", err) + klog.Fatalf("Unable to get Kubernetes versions: %v", err) } data := Data{StableVersion: stable, LatestVersion: latest} klog.Infof("Kubernetes versions: 'stable' is %s and 'latest' is %s", data.StableVersion, data.LatestVersion) diff --git a/hack/update/registry.go b/hack/update/registry.go index 5ce7bb1a93..09196494fc 100644 --- a/hack/update/registry.go +++ b/hack/update/registry.go @@ -28,7 +28,7 @@ import ( ) var ( - // keep list of registries in sync with those in "pkg/drivers/kic/types.go" + // list of registries - keep it in sync with those in "pkg/drivers/kic/types.go" registries = []registry{ { name: "Google Cloud Container Registry", @@ -51,7 +51,7 @@ var ( } ) -// container registry name, image path, credentials, and updated flag +// registry contains a container registry name, image path, and credentials. type registry struct { name string image string @@ -59,44 +59,52 @@ type registry struct { password string } -// crUpdate tags image with version, pushes it to container registry, and returns any error -func crUpdate(ctx context.Context, reg registry, image, version string) error { - login := exec.CommandContext(ctx, "docker", "login", "--username", reg.username, "--password-stdin", reg.image) - if err := RunWithRetryNotify(ctx, login, strings.NewReader(reg.password), 1*time.Minute, 10); err != nil { - return fmt.Errorf("failed logging in to %s: %w", reg.name, err) - } - klog.Infof("successfully logged in to %s", reg.name) - - tag := exec.CommandContext(ctx, "docker", "tag", image+":"+version, reg.image+":"+version) - if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Minute, 10); err != nil { - return fmt.Errorf("failed tagging %s for %s: %w", reg.image+":"+version, reg.name, err) - } - klog.Infof("successfully tagged %s for %s", reg.image+":"+version, reg.name) - - push := exec.CommandContext(ctx, "docker", "push", reg.image+":"+version) - if err := RunWithRetryNotify(ctx, push, nil, 2*time.Minute, 10); err != nil { - return fmt.Errorf("failed pushing %s to %s: %w", reg.image+":"+version, reg.name, err) - } - klog.Infof("successfully pushed %s to %s", reg.image+":"+version, reg.name) - - return nil -} - -// CRUpdateAll calls crUpdate for each available registry, and returns if at least one got updated +// CRUpdateAll updates all registries, and returns if at least one got updated. func CRUpdateAll(ctx context.Context, image, version string) (updated bool) { for _, reg := range registries { if err := crUpdate(ctx, reg, image, version); err != nil { - klog.Errorf("failed updating %s", reg.name) + klog.Errorf("Unable to update %s", reg.name) continue } - klog.Infof("successfully updated %s", reg.name) + klog.Infof("Successfully updated %s", reg.name) updated = true } return updated } -// PullImage checks if current image exists locally, tries to pull it if not, and -// returns reference image url and any error +// crUpdate tags image with version, pushes it to container registry, and returns any error occurred. +func crUpdate(ctx context.Context, reg registry, image, version string) error { + login := exec.CommandContext(ctx, "docker", "login", "--username", reg.username, "--password-stdin", reg.image) + if err := RunWithRetryNotify(ctx, login, strings.NewReader(reg.password), 1*time.Minute, 10); err != nil { + return fmt.Errorf("unable to login to %s: %w", reg.name, err) + } + klog.Infof("Successfully logged in to %s", reg.name) + + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+version, reg.image+":"+version) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Minute, 10); err != nil { + return fmt.Errorf("unable to tag %s for %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("Successfully tagged %s for %s", reg.image+":"+version, reg.name) + + push := exec.CommandContext(ctx, "docker", "push", reg.image+":"+version) + if err := RunWithRetryNotify(ctx, push, nil, 2*time.Minute, 10); err != nil { + return fmt.Errorf("unable to push %s to %s: %w", reg.image+":"+version, reg.name, err) + } + klog.Infof("Successfully pushed %s to %s", reg.image+":"+version, reg.name) + + return nil +} + +// TagImage tags local image:current with stable version, and returns any error occurred. +func TagImage(ctx context.Context, image, current, stable string) error { + tag := exec.CommandContext(ctx, "docker", "tag", image+":"+current, image+":"+stable) + if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Second, 10); err != nil { + return err + } + return nil +} + +// PullImage checks if current image exists locally, tries to pull it if not, and returns reference image url and any error occurred. func PullImage(ctx context.Context, current, release string) (image string, err error) { // check if image exists locally for _, reg := range registries { @@ -119,16 +127,7 @@ func PullImage(ctx context.Context, current, release string) (image string, err } } if image == "" { - return "", fmt.Errorf("cannot find current image version tag %s locally nor in any registry", current) + return "", fmt.Errorf("unable to find current image version tag %s locally nor in any registry", current) } return image, nil } - -// TagImage tags local image:current with stable version, and returns any error -func TagImage(ctx context.Context, image, current, stable string) error { - tag := exec.CommandContext(ctx, "docker", "tag", image+":"+current, image+":"+stable) - if err := RunWithRetryNotify(ctx, tag, nil, 1*time.Second, 10); err != nil { - return err - } - return nil -} diff --git a/hack/update/update.go b/hack/update/update.go index fc478b1563..bf1c9530d7 100644 --- a/hack/update/update.go +++ b/hack/update/update.go @@ -39,13 +39,13 @@ import ( "text/template" "time" - "k8s.io/klog/v2" - "github.com/cenkalti/backoff/v4" + + "k8s.io/klog/v2" ) const ( - // FSRoot is relative (to scripts in subfolders) root folder of local filesystem repo to update + // FSRoot is a relative (to scripts in subfolders) root folder of local filesystem repo to update FSRoot = "../../../" ) @@ -56,9 +56,11 @@ var ( // init klog and check general requirements func init() { klog.InitFlags(nil) - // write log statements to stderr instead of to files - if err := flag.Set("logtostderr", "true"); err != nil { - fmt.Printf("Error setting 'logtostderr' klog flag: %v\n", err) + if err := flag.Set("logtostderr", "false"); err != nil { + klog.Warningf("Unable to set flag value for logtostderr: %v", err) + } + if err := flag.Set("alsologtostderr", "true"); err != nil { + klog.Warningf("Unable to set flag value for alsologtostderr: %v", err) } flag.Parse() defer klog.Flush() @@ -72,21 +74,19 @@ func init() { } } -// Item defines Content where all occurrences of each Replace map key, corresponding to -// GitHub TreeEntry.Path and/or local filesystem repo file path (prefixed with FSRoot), -// would be swapped with its respective actual map value (having placeholders replaced with data), -// creating a concrete update plan. -// Replace map keys can use RegExp and map values can use Golang Text Template +// Item defines Content where all occurrences of each Replace map key, +// corresponding to GitHub TreeEntry.Path and/or local filesystem repo file path (prefixed with FSRoot), +// would be swapped with its respective actual map value (having placeholders replaced with data), creating a concrete update plan. +// Replace map keys can use RegExp and map values can use Golang Text Template. type Item struct { Content []byte `json:"-"` Replace map[string]string `json:"replace"` } -// apply updates Item Content by replacing all occurrences of Replace map's keys -// with their actual map values (with placeholders replaced with data)) +// apply updates Item Content by replacing all occurrences of Replace map's keys with their actual map values (with placeholders replaced with data). func (i *Item) apply(data interface{}) (changed bool, err error) { if i.Content == nil || i.Replace == nil { - return false, fmt.Errorf("want something, got nothing to update") + return false, fmt.Errorf("unable to update content: nothing to update") } org := string(i.Content) str := org @@ -108,18 +108,18 @@ func (i *Item) apply(data interface{}) (changed bool, err error) { func Apply(ctx context.Context, schema map[string]Item, data interface{}, prBranchPrefix, prTitle string, prIssue int) { plan, err := GetPlan(schema, data) if err != nil { - klog.Fatalf("Error parsing schema: %v\n%s", err, plan) + klog.Fatalf("Unable to parse schema: %v\n%s", err, plan) } klog.Infof("The Plan:\n%s", plan) if target == "fs" || target == "all" { changed, err := fsUpdate(FSRoot, schema, data) if err != nil { - klog.Errorf("Error updating local repo: %v", err) + klog.Errorf("Unable to update local repo: %v", err) } else if !changed { klog.Infof("Local repo update skipped: nothing changed") } else { - klog.Infof("Local repo updated") + klog.Infof("Local repo successfully updated") } } @@ -128,32 +128,31 @@ func Apply(ctx context.Context, schema map[string]Item, data interface{}, prBran tmpl := template.Must(template.New("prTitle").Parse(prTitle)) buf := new(bytes.Buffer) if err := tmpl.Execute(buf, data); err != nil { - klog.Fatalf("Error parsing PR Title: %v", err) + klog.Fatalf("Unable to parse PR Title: %v", err) } prTitle = buf.String() // check if PR already exists prURL, err := ghFindPR(ctx, prTitle, ghOwner, ghRepo, ghBase, ghToken) if err != nil { - klog.Errorf("Error checking if PR already exists: %v", err) + klog.Errorf("Unable to check if PR already exists: %v", err) } else if prURL != "" { klog.Infof("PR create skipped: already exists (%s)", prURL) } else { // create PR pr, err := ghCreatePR(ctx, ghOwner, ghRepo, ghBase, prBranchPrefix, prTitle, prIssue, ghToken, schema, data) if err != nil { - klog.Fatalf("Error creating PR: %v", err) + klog.Fatalf("Unable to create PR: %v", err) } else if pr == nil { klog.Infof("PR create skipped: nothing changed") } else { - klog.Infof("PR created: %s", *pr.HTMLURL) + klog.Infof("PR successfully created: %s", *pr.HTMLURL) } } } } -// GetPlan returns concrete plan replacing placeholders in schema with actual data values, -// returns JSON-formatted representation of the plan and any error +// GetPlan returns concrete plan replacing placeholders in schema with actual data values, returns JSON-formatted representation of the plan and any error occurred. func GetPlan(schema map[string]Item, data interface{}) (prettyprint string, err error) { for _, item := range schema { for src, dst := range item.Replace {