Boilerplate checking python script replaced with go

.travis.yml

@@ -7,12 +7,10 @@ env:
     - GOPROXY=https://proxy.golang.org
 matrix:
   include:
-    - language: python
+    - language: go
       name: Check Boilerplate 
       env:
         - TESTSUITE=boilerplate
-      before_install: 
-          - pip install flake8 && flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
       script: make test
 
     - language: go

hack/boilerplate/boilerplate.go

@@ -0,0 +1,157 @@
+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+var skippedDirs = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go`)
+
+func main() {
+	rootdir, _ := os.Getwd()
+	rootdir += "/../../"
+	var boilerplateDir string
+	var filenames []string
+
+	for i := 1; i < len(os.Args); i++ {
+		re := regexp.MustCompile("^--.*")
+		if re.MatchString(os.Args[i]) {
+			if os.Args[i] == "--rootdir" {
+				rootdir = os.Args[i+1]
+				i++
+				continue
+			}
+			if os.Args[i] == "--boilerplate-dir" {
+				boilerplateDir = os.Args[i+1]
+				i++
+				continue
+			}
+		} else {
+			filenames = append(filenames, os.Args[i])
+		}
+
+	}
+
+	if len(boilerplateDir) == 0 {
+		boilerplateDir = filepath.Join(rootdir, "hack/boilerplate")
+	}
+
+	refs := getRefs(boilerplateDir)
+	files := getFileList(rootdir, refs, filenames)
+	for _, file := range files {
+		if !filePasses(file, refs[getFileExtension(file)]) {
+			fmt.Println(file)
+		}
+	}
+
+}
+
+func getRefs(dir string) map[string][]byte {
+	refs := make(map[string][]byte)
+	files, _ := filepath.Glob(dir + "/*.txt")
+	for _, filename := range files {
+		extension := strings.ToLower(strings.Split(filename, ".")[1])
+		data, err := ioutil.ReadFile(filename)
+		if err != nil {
+			log.Fatal(err)
+		}
+		re := regexp.MustCompile(`\r`)
+		refs[extension] = re.ReplaceAll(data, nil)
+	}
+	return refs
+}
+
+func filePasses(filename string, ref []byte) bool {
+	var re *regexp.Regexp
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		log.Fatal(err)
+	}
+	re = regexp.MustCompile(`\r`)
+	data = re.ReplaceAll(data, nil)
+
+	extension := getFileExtension(filename)
+
+	// remove build tags from the top of Go files
+	if extension == "go" {
+		// \r is necessary for windows file endings
+		re = regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`)
+		data = re.ReplaceAll(data, nil)
+	}
+
+	// remove shebang from the top of shell files
+	if extension == "sh" {
+		// \r is necessary for windows file endings
+		// re := regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`)
+		re = regexp.MustCompile(`(?m)^(#!.*\r{0,1}\n)(\r{0,1}\n)*`)
+		data = re.ReplaceAll(data, nil)
+	}
+
+	// if our test file is smaller than the reference it surely fails!
+	if len(data) < len(ref) {
+		return false
+	}
+
+	data = data[:len(ref)]
+
+	// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
+	re = regexp.MustCompile(`Copyright YEAR`)
+	if re.Match(data) {
+		return false
+	}
+
+	// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
+	re = regexp.MustCompile(`Copyright \d{4}`)
+	data = re.ReplaceAll(data, []byte(`Copyright YEAR`))
+
+	return bytes.Equal(data, ref)
+}
+
+// get the file extensin or the filename if the file has no extension
+func getFileExtension(filename string) string {
+	splitted := strings.Split(filepath.Base(filename), ".")
+	return strings.ToLower(splitted[len(splitted)-1])
+}
+
+func getFileList(rootDir string, extensions map[string][]byte, files []string) []string {
+	var outFiles []string
+	if len(files) == 0 {
+		err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
+			// println(path)
+			if !info.IsDir() && !skippedDirs.MatchString(filepath.Dir(path)) {
+				if extensions[strings.ToLower(getFileExtension(path))] != nil {
+					outFiles = append(outFiles, path)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		outFiles = files
+	}
+	return outFiles
+}

hack/boilerplate/boilerplate.py

@@ -1,170 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright 2015 The Kubernetes Authors All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from __future__ import print_function
-
-import argparse
-import glob
-import json
-import mmap
-import os
-import re
-import sys
-
-parser = argparse.ArgumentParser()
-parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*')
-
-rootdir = os.path.dirname(__file__) + "/../../"
-rootdir = os.path.abspath(rootdir)
-parser.add_argument("--rootdir", default=rootdir, help="root directory to examine")
-
-default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
-parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir)
-args = parser.parse_args()
-
-
-def get_refs():
-    refs = {}
-
-    for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
-        extension = os.path.basename(path).split(".")[1]
-
-        ref_file = open(path, 'r')
-        ref = ref_file.read().splitlines()
-        ref_file.close()
-        refs[extension] = ref
-
-    return refs
-
-def file_passes(filename, refs, regexs):
-    try:
-        f = open(filename, 'r')
-    except:
-        return False
-
-    data = f.read()
-    f.close()
-
-    basename = os.path.basename(filename)
-    extension = file_extension(filename)
-    if extension != "":
-        ref = refs[extension]
-    else:
-        ref = refs[basename]
-
-    # remove build tags from the top of Go files
-    if extension == "go":
-        p = regexs["go_build_constraints"]
-        (data, found) = p.subn("", data, 1)
-
-    # remove shebang from the top of shell files
-    if extension == "sh":
-        p = regexs["shebang"]
-        (data, found) = p.subn("", data, 1)
-
-    data = data.splitlines()
-
-    # if our test file is smaller than the reference it surely fails!
-    if len(ref) > len(data):
-        return False
-
-    # trim our file to the same number of lines as the reference file
-    data = data[:len(ref)]
-
-    p = regexs["year"]
-    for d in data:
-        if p.search(d):
-            return False
-
-    # Replace all occurrences of the regex "2018|2017|2016|2015|2014" with "YEAR"
-    p = regexs["date"]
-    for i, d in enumerate(data):
-        (data[i], found) = p.subn('YEAR', d)
-        if found != 0:
-            break
-
-    # if we don't match the reference at this point, fail
-    if ref != data:
-        return False
-
-    return True
-
-def file_extension(filename):
-    return os.path.splitext(filename)[1].split(".")[-1].lower()
-
-skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh', "vendor", "test/e2e/generated/bindata.go"]
-
-def normalize_files(files):
-    newfiles = []
-    for pathname in files:
-        if any(x in pathname for x in skipped_dirs):
-            continue
-        newfiles.append(pathname)
-    for i, pathname in enumerate(newfiles):
-        if not os.path.isabs(pathname):
-            newfiles[i] = os.path.join(rootdir, pathname)
-    return newfiles
-
-def get_files(extensions):
-    files = []
-    if len(args.filenames) > 0:
-        files = args.filenames
-    else:
-        for root, dirs, walkfiles in os.walk(args.rootdir):
-            # don't visit certain dirs. This is just a performance improvement
-            # as we would prune these later in normalize_files(). But doing it
-            # cuts down the amount of filesystem walking we do and cuts down
-            # the size of the file list
-            for d in skipped_dirs:
-                if d in dirs:
-                    dirs.remove(d)
-
-            for name in walkfiles:
-                pathname = os.path.join(root, name)
-                files.append(pathname)
-
-    files = normalize_files(files)
-    outfiles = []
-    for pathname in files:
-        basename = os.path.basename(pathname)
-        extension = file_extension(pathname)
-        if extension in extensions or basename in extensions:
-            outfiles.append(pathname)
-    return outfiles
-
-def get_regexs():
-    regexs = {}
-    # Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
-    regexs["year"] = re.compile( 'YEAR' )
-    # dates can be 2010 to 2039
-    regexs["date"] = re.compile( '(20[123]\d)' )
-    # strip // +build \n\n build constraints
-    regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
-    # strip #!.* from shell scripts
-    regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
-    return regexs
-
-def main():
-    regexs = get_regexs()
-    refs = get_refs()
-    filenames = get_files(refs.keys())
-
-    for filename in filenames:
-        if not file_passes(filename, refs, regexs):
-            print(filename, file=sys.stdout)
-
-if __name__ == "__main__":
-  sys.exit(main())

hack/boilerplate/fix.sh

@@ -21,7 +21,7 @@ function prepend() {
   local pattern=$1
   local ref=$2
   local headers=$3
-  local files=$(hack/boilerplate/boilerplate.py --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern")
+  local files=$(hack/boilerplate/boilerplate --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern")
   for f in ${files}; do
     echo ${f};
     local copyright="$(cat hack/boilerplate/boilerplate.${ref}.txt | sed s/YEAR/$(date +%Y)/g)"

hack/boilerplate/go.mod

@@ -0,0 +1,3 @@
+module k8s.io/minikube/hack/boilerplate
+
+go 1.12

test.sh

@@ -33,9 +33,11 @@ fi
 if [[ "$TESTSUITE" = "boilerplate" ]] || [[ "$TESTSUITE" = "all" ]]
 then
     echo "= boilerplate ==========================================================="
-    readonly PYTHON=$(type -P python || echo docker run --rm -it -v $(pwd):/minikube -w /minikube python python)
     readonly BDIR="./hack/boilerplate"
-    missing="$($PYTHON ${BDIR}/boilerplate.py --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)"
+    pushd ${BDIR}
+	go build
+	popd
+	missing="$(${BDIR}/boilerplate --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)"
     if [[ -n "${missing}" ]]; then
         echo "boilerplate missing: $missing"
         echo "consider running: ${BDIR}/fix.sh"