diff --git a/.travis.yml b/.travis.yml index 289137e66e..3f84bd8a8d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,12 +7,10 @@ env: - GOPROXY=https://proxy.golang.org matrix: include: - - language: python + - language: go name: Check Boilerplate env: - TESTSUITE=boilerplate - before_install: - - pip install flake8 && flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics script: make test - language: go diff --git a/hack/boilerplate/boilerplate.go b/hack/boilerplate/boilerplate.go new file mode 100644 index 0000000000..f250d23f16 --- /dev/null +++ b/hack/boilerplate/boilerplate.go @@ -0,0 +1,157 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "path/filepath" + "regexp" + "strings" +) + +var skippedDirs = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go`) + +func main() { + rootdir, _ := os.Getwd() + rootdir += "/../../" + var boilerplateDir string + var filenames []string + + for i := 1; i < len(os.Args); i++ { + re := regexp.MustCompile("^--.*") + if re.MatchString(os.Args[i]) { + if os.Args[i] == "--rootdir" { + rootdir = os.Args[i+1] + i++ + continue + } + if os.Args[i] == "--boilerplate-dir" { + boilerplateDir = os.Args[i+1] + i++ + continue + } + } else { + filenames = append(filenames, os.Args[i]) + } + + } + + if len(boilerplateDir) == 0 { + boilerplateDir = filepath.Join(rootdir, "hack/boilerplate") + } + + refs := getRefs(boilerplateDir) + files := getFileList(rootdir, refs, filenames) + for _, file := range files { + if !filePasses(file, refs[getFileExtension(file)]) { + fmt.Println(file) + } + } + +} + +func getRefs(dir string) map[string][]byte { + refs := make(map[string][]byte) + files, _ := filepath.Glob(dir + "/*.txt") + for _, filename := range files { + extension := strings.ToLower(strings.Split(filename, ".")[1]) + data, err := ioutil.ReadFile(filename) + if err != nil { + log.Fatal(err) + } + re := regexp.MustCompile(`\r`) + refs[extension] = re.ReplaceAll(data, nil) + } + return refs +} + +func filePasses(filename string, ref []byte) bool { + var re *regexp.Regexp + data, err := ioutil.ReadFile(filename) + if err != nil { + log.Fatal(err) + } + re = regexp.MustCompile(`\r`) + data = re.ReplaceAll(data, nil) + + extension := getFileExtension(filename) + + // remove build tags from the top of Go files + if extension == "go" { + // \r is necessary for windows file endings + re = regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`) + data = re.ReplaceAll(data, nil) + } + + // remove shebang from the top of shell files + if extension == "sh" { + // \r is necessary for windows file endings + // re := regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`) + re = regexp.MustCompile(`(?m)^(#!.*\r{0,1}\n)(\r{0,1}\n)*`) + data = re.ReplaceAll(data, nil) + } + + // if our test file is smaller than the reference it surely fails! + if len(data) < len(ref) { + return false + } + + data = data[:len(ref)] + + // Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing + re = regexp.MustCompile(`Copyright YEAR`) + if re.Match(data) { + return false + } + + // Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR" + re = regexp.MustCompile(`Copyright \d{4}`) + data = re.ReplaceAll(data, []byte(`Copyright YEAR`)) + + return bytes.Equal(data, ref) +} + +// get the file extensin or the filename if the file has no extension +func getFileExtension(filename string) string { + splitted := strings.Split(filepath.Base(filename), ".") + return strings.ToLower(splitted[len(splitted)-1]) +} + +func getFileList(rootDir string, extensions map[string][]byte, files []string) []string { + var outFiles []string + if len(files) == 0 { + err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error { + // println(path) + if !info.IsDir() && !skippedDirs.MatchString(filepath.Dir(path)) { + if extensions[strings.ToLower(getFileExtension(path))] != nil { + outFiles = append(outFiles, path) + } + } + return nil + }) + if err != nil { + log.Fatal(err) + } + } else { + outFiles = files + } + return outFiles +} diff --git a/hack/boilerplate/boilerplate.py b/hack/boilerplate/boilerplate.py deleted file mode 100755 index 35fc425706..0000000000 --- a/hack/boilerplate/boilerplate.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python - -# Copyright 2015 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import argparse -import glob -import json -import mmap -import os -import re -import sys - -parser = argparse.ArgumentParser() -parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*') - -rootdir = os.path.dirname(__file__) + "/../../" -rootdir = os.path.abspath(rootdir) -parser.add_argument("--rootdir", default=rootdir, help="root directory to examine") - -default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate") -parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir) -args = parser.parse_args() - - -def get_refs(): - refs = {} - - for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")): - extension = os.path.basename(path).split(".")[1] - - ref_file = open(path, 'r') - ref = ref_file.read().splitlines() - ref_file.close() - refs[extension] = ref - - return refs - -def file_passes(filename, refs, regexs): - try: - f = open(filename, 'r') - except: - return False - - data = f.read() - f.close() - - basename = os.path.basename(filename) - extension = file_extension(filename) - if extension != "": - ref = refs[extension] - else: - ref = refs[basename] - - # remove build tags from the top of Go files - if extension == "go": - p = regexs["go_build_constraints"] - (data, found) = p.subn("", data, 1) - - # remove shebang from the top of shell files - if extension == "sh": - p = regexs["shebang"] - (data, found) = p.subn("", data, 1) - - data = data.splitlines() - - # if our test file is smaller than the reference it surely fails! - if len(ref) > len(data): - return False - - # trim our file to the same number of lines as the reference file - data = data[:len(ref)] - - p = regexs["year"] - for d in data: - if p.search(d): - return False - - # Replace all occurrences of the regex "2018|2017|2016|2015|2014" with "YEAR" - p = regexs["date"] - for i, d in enumerate(data): - (data[i], found) = p.subn('YEAR', d) - if found != 0: - break - - # if we don't match the reference at this point, fail - if ref != data: - return False - - return True - -def file_extension(filename): - return os.path.splitext(filename)[1].split(".")[-1].lower() - -skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh', "vendor", "test/e2e/generated/bindata.go"] - -def normalize_files(files): - newfiles = [] - for pathname in files: - if any(x in pathname for x in skipped_dirs): - continue - newfiles.append(pathname) - for i, pathname in enumerate(newfiles): - if not os.path.isabs(pathname): - newfiles[i] = os.path.join(rootdir, pathname) - return newfiles - -def get_files(extensions): - files = [] - if len(args.filenames) > 0: - files = args.filenames - else: - for root, dirs, walkfiles in os.walk(args.rootdir): - # don't visit certain dirs. This is just a performance improvement - # as we would prune these later in normalize_files(). But doing it - # cuts down the amount of filesystem walking we do and cuts down - # the size of the file list - for d in skipped_dirs: - if d in dirs: - dirs.remove(d) - - for name in walkfiles: - pathname = os.path.join(root, name) - files.append(pathname) - - files = normalize_files(files) - outfiles = [] - for pathname in files: - basename = os.path.basename(pathname) - extension = file_extension(pathname) - if extension in extensions or basename in extensions: - outfiles.append(pathname) - return outfiles - -def get_regexs(): - regexs = {} - # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing - regexs["year"] = re.compile( 'YEAR' ) - # dates can be 2010 to 2039 - regexs["date"] = re.compile( '(20[123]\d)' ) - # strip // +build \n\n build constraints - regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE) - # strip #!.* from shell scripts - regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE) - return regexs - -def main(): - regexs = get_regexs() - refs = get_refs() - filenames = get_files(refs.keys()) - - for filename in filenames: - if not file_passes(filename, refs, regexs): - print(filename, file=sys.stdout) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/hack/boilerplate/fix.sh b/hack/boilerplate/fix.sh index 4d01cb312c..177bda08f5 100755 --- a/hack/boilerplate/fix.sh +++ b/hack/boilerplate/fix.sh @@ -21,7 +21,7 @@ function prepend() { local pattern=$1 local ref=$2 local headers=$3 - local files=$(hack/boilerplate/boilerplate.py --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern") + local files=$(hack/boilerplate/boilerplate --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern") for f in ${files}; do echo ${f}; local copyright="$(cat hack/boilerplate/boilerplate.${ref}.txt | sed s/YEAR/$(date +%Y)/g)" diff --git a/hack/boilerplate/go.mod b/hack/boilerplate/go.mod new file mode 100644 index 0000000000..d82a0cd2d1 --- /dev/null +++ b/hack/boilerplate/go.mod @@ -0,0 +1,3 @@ +module k8s.io/minikube/hack/boilerplate + +go 1.12 diff --git a/test.sh b/test.sh index c6173fb5b8..1e5c81b80b 100755 --- a/test.sh +++ b/test.sh @@ -33,9 +33,11 @@ fi if [[ "$TESTSUITE" = "boilerplate" ]] || [[ "$TESTSUITE" = "all" ]] then echo "= boilerplate ===========================================================" - readonly PYTHON=$(type -P python || echo docker run --rm -it -v $(pwd):/minikube -w /minikube python python) readonly BDIR="./hack/boilerplate" - missing="$($PYTHON ${BDIR}/boilerplate.py --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)" + pushd ${BDIR} + go build + popd + missing="$(${BDIR}/boilerplate --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)" if [[ -n "${missing}" ]]; then echo "boilerplate missing: $missing" echo "consider running: ${BDIR}/fix.sh"