Boilerplate checking python script replaced with go
duohedron
parent
ce7babf78c
commit
bcd7d73389
|
|
@ -7,12 +7,10 @@ env:
|
|||
- GOPROXY=https://proxy.golang.org
|
||||
matrix:
|
||||
include:
|
||||
- language: python
|
||||
- language: go
|
||||
name: Check Boilerplate
|
||||
env:
|
||||
- TESTSUITE=boilerplate
|
||||
before_install:
|
||||
- pip install flake8 && flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
script: make test
|
||||
|
||||
- language: go
|
||||
|
|
|
|||
|
|
@ -0,0 +1,157 @@
|
|||
/*
|
||||
Copyright 2019 The Kubernetes Authors All rights reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var skippedDirs = regexp.MustCompile(`Godeps|third_party|_gopath|_output|\.git|cluster/env.sh|vendor|test/e2e/generated/bindata.go`)
|
||||
|
||||
func main() {
|
||||
rootdir, _ := os.Getwd()
|
||||
rootdir += "/../../"
|
||||
var boilerplateDir string
|
||||
var filenames []string
|
||||
|
||||
for i := 1; i < len(os.Args); i++ {
|
||||
re := regexp.MustCompile("^--.*")
|
||||
if re.MatchString(os.Args[i]) {
|
||||
if os.Args[i] == "--rootdir" {
|
||||
rootdir = os.Args[i+1]
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if os.Args[i] == "--boilerplate-dir" {
|
||||
boilerplateDir = os.Args[i+1]
|
||||
i++
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
filenames = append(filenames, os.Args[i])
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if len(boilerplateDir) == 0 {
|
||||
boilerplateDir = filepath.Join(rootdir, "hack/boilerplate")
|
||||
}
|
||||
|
||||
refs := getRefs(boilerplateDir)
|
||||
files := getFileList(rootdir, refs, filenames)
|
||||
for _, file := range files {
|
||||
if !filePasses(file, refs[getFileExtension(file)]) {
|
||||
fmt.Println(file)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getRefs(dir string) map[string][]byte {
|
||||
refs := make(map[string][]byte)
|
||||
files, _ := filepath.Glob(dir + "/*.txt")
|
||||
for _, filename := range files {
|
||||
extension := strings.ToLower(strings.Split(filename, ".")[1])
|
||||
data, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
re := regexp.MustCompile(`\r`)
|
||||
refs[extension] = re.ReplaceAll(data, nil)
|
||||
}
|
||||
return refs
|
||||
}
|
||||
|
||||
func filePasses(filename string, ref []byte) bool {
|
||||
var re *regexp.Regexp
|
||||
data, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
re = regexp.MustCompile(`\r`)
|
||||
data = re.ReplaceAll(data, nil)
|
||||
|
||||
extension := getFileExtension(filename)
|
||||
|
||||
// remove build tags from the top of Go files
|
||||
if extension == "go" {
|
||||
// \r is necessary for windows file endings
|
||||
re = regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`)
|
||||
data = re.ReplaceAll(data, nil)
|
||||
}
|
||||
|
||||
// remove shebang from the top of shell files
|
||||
if extension == "sh" {
|
||||
// \r is necessary for windows file endings
|
||||
// re := regexp.MustCompile(`(?m)^(// \+build.*\r{0,1}\n)+\r{0,1}\n`)
|
||||
re = regexp.MustCompile(`(?m)^(#!.*\r{0,1}\n)(\r{0,1}\n)*`)
|
||||
data = re.ReplaceAll(data, nil)
|
||||
}
|
||||
|
||||
// if our test file is smaller than the reference it surely fails!
|
||||
if len(data) < len(ref) {
|
||||
return false
|
||||
}
|
||||
|
||||
data = data[:len(ref)]
|
||||
|
||||
// Search for "Copyright YEAR" which exists in the boilerplate, but shouldn't in the real thing
|
||||
re = regexp.MustCompile(`Copyright YEAR`)
|
||||
if re.Match(data) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Replace all occurrences of the regex "Copyright \d{4}" with "Copyright YEAR"
|
||||
re = regexp.MustCompile(`Copyright \d{4}`)
|
||||
data = re.ReplaceAll(data, []byte(`Copyright YEAR`))
|
||||
|
||||
return bytes.Equal(data, ref)
|
||||
}
|
||||
|
||||
// get the file extensin or the filename if the file has no extension
|
||||
func getFileExtension(filename string) string {
|
||||
splitted := strings.Split(filepath.Base(filename), ".")
|
||||
return strings.ToLower(splitted[len(splitted)-1])
|
||||
}
|
||||
|
||||
func getFileList(rootDir string, extensions map[string][]byte, files []string) []string {
|
||||
var outFiles []string
|
||||
if len(files) == 0 {
|
||||
err := filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
|
||||
// println(path)
|
||||
if !info.IsDir() && !skippedDirs.MatchString(filepath.Dir(path)) {
|
||||
if extensions[strings.ToLower(getFileExtension(path))] != nil {
|
||||
outFiles = append(outFiles, path)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
} else {
|
||||
outFiles = files
|
||||
}
|
||||
return outFiles
|
||||
}
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2015 The Kubernetes Authors All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import mmap
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("filenames", help="list of files to check, all files if unspecified", nargs='*')
|
||||
|
||||
rootdir = os.path.dirname(__file__) + "/../../"
|
||||
rootdir = os.path.abspath(rootdir)
|
||||
parser.add_argument("--rootdir", default=rootdir, help="root directory to examine")
|
||||
|
||||
default_boilerplate_dir = os.path.join(rootdir, "hack/boilerplate")
|
||||
parser.add_argument("--boilerplate-dir", default=default_boilerplate_dir)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
def get_refs():
|
||||
refs = {}
|
||||
|
||||
for path in glob.glob(os.path.join(args.boilerplate_dir, "boilerplate.*.txt")):
|
||||
extension = os.path.basename(path).split(".")[1]
|
||||
|
||||
ref_file = open(path, 'r')
|
||||
ref = ref_file.read().splitlines()
|
||||
ref_file.close()
|
||||
refs[extension] = ref
|
||||
|
||||
return refs
|
||||
|
||||
def file_passes(filename, refs, regexs):
|
||||
try:
|
||||
f = open(filename, 'r')
|
||||
except:
|
||||
return False
|
||||
|
||||
data = f.read()
|
||||
f.close()
|
||||
|
||||
basename = os.path.basename(filename)
|
||||
extension = file_extension(filename)
|
||||
if extension != "":
|
||||
ref = refs[extension]
|
||||
else:
|
||||
ref = refs[basename]
|
||||
|
||||
# remove build tags from the top of Go files
|
||||
if extension == "go":
|
||||
p = regexs["go_build_constraints"]
|
||||
(data, found) = p.subn("", data, 1)
|
||||
|
||||
# remove shebang from the top of shell files
|
||||
if extension == "sh":
|
||||
p = regexs["shebang"]
|
||||
(data, found) = p.subn("", data, 1)
|
||||
|
||||
data = data.splitlines()
|
||||
|
||||
# if our test file is smaller than the reference it surely fails!
|
||||
if len(ref) > len(data):
|
||||
return False
|
||||
|
||||
# trim our file to the same number of lines as the reference file
|
||||
data = data[:len(ref)]
|
||||
|
||||
p = regexs["year"]
|
||||
for d in data:
|
||||
if p.search(d):
|
||||
return False
|
||||
|
||||
# Replace all occurrences of the regex "2018|2017|2016|2015|2014" with "YEAR"
|
||||
p = regexs["date"]
|
||||
for i, d in enumerate(data):
|
||||
(data[i], found) = p.subn('YEAR', d)
|
||||
if found != 0:
|
||||
break
|
||||
|
||||
# if we don't match the reference at this point, fail
|
||||
if ref != data:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def file_extension(filename):
|
||||
return os.path.splitext(filename)[1].split(".")[-1].lower()
|
||||
|
||||
skipped_dirs = ['Godeps', 'third_party', '_gopath', '_output', '.git', 'cluster/env.sh', "vendor", "test/e2e/generated/bindata.go"]
|
||||
|
||||
def normalize_files(files):
|
||||
newfiles = []
|
||||
for pathname in files:
|
||||
if any(x in pathname for x in skipped_dirs):
|
||||
continue
|
||||
newfiles.append(pathname)
|
||||
for i, pathname in enumerate(newfiles):
|
||||
if not os.path.isabs(pathname):
|
||||
newfiles[i] = os.path.join(rootdir, pathname)
|
||||
return newfiles
|
||||
|
||||
def get_files(extensions):
|
||||
files = []
|
||||
if len(args.filenames) > 0:
|
||||
files = args.filenames
|
||||
else:
|
||||
for root, dirs, walkfiles in os.walk(args.rootdir):
|
||||
# don't visit certain dirs. This is just a performance improvement
|
||||
# as we would prune these later in normalize_files(). But doing it
|
||||
# cuts down the amount of filesystem walking we do and cuts down
|
||||
# the size of the file list
|
||||
for d in skipped_dirs:
|
||||
if d in dirs:
|
||||
dirs.remove(d)
|
||||
|
||||
for name in walkfiles:
|
||||
pathname = os.path.join(root, name)
|
||||
files.append(pathname)
|
||||
|
||||
files = normalize_files(files)
|
||||
outfiles = []
|
||||
for pathname in files:
|
||||
basename = os.path.basename(pathname)
|
||||
extension = file_extension(pathname)
|
||||
if extension in extensions or basename in extensions:
|
||||
outfiles.append(pathname)
|
||||
return outfiles
|
||||
|
||||
def get_regexs():
|
||||
regexs = {}
|
||||
# Search for "YEAR" which exists in the boilerplate, but shouldn't in the real thing
|
||||
regexs["year"] = re.compile( 'YEAR' )
|
||||
# dates can be 2010 to 2039
|
||||
regexs["date"] = re.compile( '(20[123]\d)' )
|
||||
# strip // +build \n\n build constraints
|
||||
regexs["go_build_constraints"] = re.compile(r"^(// \+build.*\n)+\n", re.MULTILINE)
|
||||
# strip #!.* from shell scripts
|
||||
regexs["shebang"] = re.compile(r"^(#!.*\n)\n*", re.MULTILINE)
|
||||
return regexs
|
||||
|
||||
def main():
|
||||
regexs = get_regexs()
|
||||
refs = get_refs()
|
||||
filenames = get_files(refs.keys())
|
||||
|
||||
for filename in filenames:
|
||||
if not file_passes(filename, refs, regexs):
|
||||
print(filename, file=sys.stdout)
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
|
|
@ -21,7 +21,7 @@ function prepend() {
|
|||
local pattern=$1
|
||||
local ref=$2
|
||||
local headers=$3
|
||||
local files=$(hack/boilerplate/boilerplate.py --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern")
|
||||
local files=$(hack/boilerplate/boilerplate --rootdir ${ROOT_DIR} | grep -v "$ignore" | grep "$pattern")
|
||||
for f in ${files}; do
|
||||
echo ${f};
|
||||
local copyright="$(cat hack/boilerplate/boilerplate.${ref}.txt | sed s/YEAR/$(date +%Y)/g)"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
module k8s.io/minikube/hack/boilerplate
|
||||
|
||||
go 1.12
|
||||
6
test.sh
6
test.sh
|
|
@ -33,9 +33,11 @@ fi
|
|||
if [[ "$TESTSUITE" = "boilerplate" ]] || [[ "$TESTSUITE" = "all" ]]
|
||||
then
|
||||
echo "= boilerplate ==========================================================="
|
||||
readonly PYTHON=$(type -P python || echo docker run --rm -it -v $(pwd):/minikube -w /minikube python python)
|
||||
readonly BDIR="./hack/boilerplate"
|
||||
missing="$($PYTHON ${BDIR}/boilerplate.py --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)"
|
||||
pushd ${BDIR}
|
||||
go build
|
||||
popd
|
||||
missing="$(${BDIR}/boilerplate --rootdir . --boilerplate-dir ${BDIR} | egrep -v '/assets.go|/translations.go|/site/themes/|/site/node_modules|\./out|/hugo/' || true)"
|
||||
if [[ -n "${missing}" ]]; then
|
||||
echo "boilerplate missing: $missing"
|
||||
echo "consider running: ${BDIR}/fix.sh"
|
||||
|
|
|
|||
Loading…
Reference in New Issue