From 618c212c32f792e60becb7b3168e13a537135315 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 24 Aug 2017 09:36:36 -0700 Subject: [PATCH] Move mungedocs tool to the docs repo. (#5178) * Remove seemingly errant file * Move mungedocs tool to docs repo * Bump go version to 1.8.3 --- .travis.yml | 6 +- _tools/mungedocs/README.md | 22 ++ _tools/mungedocs/analytics.go | 58 ++++ _tools/mungedocs/analytics_test.go | 94 ++++++ _tools/mungedocs/example_syncer.go | 121 ++++++++ _tools/mungedocs/example_syncer_test.go | 68 ++++ _tools/mungedocs/headers.go | 74 +++++ _tools/mungedocs/headers_test.go | 74 +++++ _tools/mungedocs/kubectl_dash_f.go | 125 ++++++++ _tools/mungedocs/kubectl_dash_f_test.go | 143 +++++++++ _tools/mungedocs/links.go | 238 ++++++++++++++ _tools/mungedocs/links_test.go | 75 +++++ _tools/mungedocs/mungedocs.go | 235 ++++++++++++++ _tools/mungedocs/preformatted.go | 51 +++ _tools/mungedocs/preformatted_test.go | 98 ++++++ _tools/mungedocs/testdata/README.md | 1 + _tools/mungedocs/testdata/example.txt | 1 + _tools/mungedocs/testdata/pod.yaml | 10 + _tools/mungedocs/testdata/test-dashes.md | 1 + _tools/mungedocs/testdata/test_underscores.md | 1 + _tools/mungedocs/toc.go | 89 ++++++ _tools/mungedocs/toc_test.go | 79 +++++ _tools/mungedocs/util.go | 291 ++++++++++++++++++ _tools/mungedocs/util_test.go | 172 +++++++++++ _tools/mungedocs/whitespace.go | 31 ++ _tools/mungedocs/whitespace_test.go | 46 +++ quota-pod-deployment.yaml | 14 - 27 files changed, 2201 insertions(+), 17 deletions(-) create mode 100644 _tools/mungedocs/README.md create mode 100644 _tools/mungedocs/analytics.go create mode 100644 _tools/mungedocs/analytics_test.go create mode 100644 _tools/mungedocs/example_syncer.go create mode 100644 _tools/mungedocs/example_syncer_test.go create mode 100644 _tools/mungedocs/headers.go create mode 100644 _tools/mungedocs/headers_test.go create mode 100644 _tools/mungedocs/kubectl_dash_f.go create mode 100644 _tools/mungedocs/kubectl_dash_f_test.go create mode 100644 _tools/mungedocs/links.go create mode 100644 _tools/mungedocs/links_test.go create mode 100644 _tools/mungedocs/mungedocs.go create mode 100644 _tools/mungedocs/preformatted.go create mode 100644 _tools/mungedocs/preformatted_test.go create mode 100644 _tools/mungedocs/testdata/README.md create mode 100644 _tools/mungedocs/testdata/example.txt create mode 100644 _tools/mungedocs/testdata/pod.yaml create mode 100644 _tools/mungedocs/testdata/test-dashes.md create mode 100644 _tools/mungedocs/testdata/test_underscores.md create mode 100644 _tools/mungedocs/toc.go create mode 100644 _tools/mungedocs/toc_test.go create mode 100644 _tools/mungedocs/util.go create mode 100644 _tools/mungedocs/util_test.go create mode 100644 _tools/mungedocs/whitespace.go create mode 100644 _tools/mungedocs/whitespace_test.go delete mode 100644 quota-pod-deployment.yaml diff --git a/.travis.yml b/.travis.yml index 6d889f3c23..964b903136 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.8.1 + - 1.8.3 # Don't want default ./... here: install: @@ -19,8 +19,8 @@ install: - git clone --depth=50 --branch=master https://github.com/kubernetes/md-check $HOME/gopath/src/k8s.io/md-check - go get -t -v k8s.io/md-check -# (3) Fetch mungedocs -- go get -v k8s.io/kubernetes/cmd/mungedocs +# (3) Build mungedocs +- go install ./_tools/mungedocs script: - go test -v k8s.io/kubernetes.github.io/test diff --git a/_tools/mungedocs/README.md b/_tools/mungedocs/README.md new file mode 100644 index 0000000000..5fe3ed106c --- /dev/null +++ b/_tools/mungedocs/README.md @@ -0,0 +1,22 @@ +# Documentation Mungers + +Basically this is like lint/gofmt for md docs. + +It basically does the following: +- iterate over all files in the given doc root. +- for each file split it into a slice (mungeLines) of lines (mungeLine) +- a mungeline has metadata about each line typically determined by a 'fast' regex. + - metadata contains things like 'is inside a preformatted block' + - contains a markdown header + - has a link to another file + - etc.. + - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one. +- each munger is then called in turn + - they are given the mungeLines + - they create an entirely new set of mungeLines with their modifications + - the new set is returned +- the new set is then fed into the next munger. +- in the end we might commit the end mungeLines to the file or not (--verify) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]() diff --git a/_tools/mungedocs/analytics.go b/_tools/mungedocs/analytics.go new file mode 100644 index 0000000000..a7eaefa080 --- /dev/null +++ b/_tools/mungedocs/analytics.go @@ -0,0 +1,58 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" +) + +const analyticsMungeTag = "GENERATED_ANALYTICS" +const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/" + +func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + fileName, err := makeRepoRelative(fileName, fileName) + if err != nil { + return mlines, err + } + + link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName) + insertLines := getMungeLines(link) + mlines, err = removeMacroBlock(analyticsMungeTag, mlines) + if err != nil { + return mlines, err + } + + // Remove floating analytics links not surrounded by the munge tags. + for _, mline := range mlines { + if mline.preformatted || mline.header || mline.beginTag || mline.endTag { + out = append(out, mline) + continue + } + if strings.HasPrefix(mline.data, analyticsLinePrefix) { + continue + } + out = append(out, mline) + } + out = appendMacroBlock(out, analyticsMungeTag) + out, err = updateMacroBlock(out, analyticsMungeTag, insertLines) + if err != nil { + return mlines, err + } + return out, nil +} diff --git a/_tools/mungedocs/analytics_test.go b/_tools/mungedocs/analytics_test.go new file mode 100644 index 0000000000..99306f2fe0 --- /dev/null +++ b/_tools/mungedocs/analytics_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestAnalytics(t *testing.T) { + b := beginMungeTag("GENERATED_ANALYTICS") + e := endMungeTag("GENERATED_ANALYTICS") + var cases = []struct { + in string + expected string + }{ + { + "aoeu", + "aoeu" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()", + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "prefix" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + + "\n" + "suffix", + "prefix" + "\n" + "suffix" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + out, err := updateAnalytics("path/to/file-name.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !expected.Equal(out) { + t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String()) + } + } +} diff --git a/_tools/mungedocs/example_syncer.go b/_tools/mungedocs/example_syncer.go new file mode 100644 index 0000000000..c15255be43 --- /dev/null +++ b/_tools/mungedocs/example_syncer.go @@ -0,0 +1,121 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io/ioutil" + "regexp" + "strings" +) + +const exampleToken = "EXAMPLE" + +const exampleLineStart = " +// +// ```yaml +// foo: +// bar: +// ``` +// +// [Download example](../../examples/guestbook/frontend-service.yaml?raw=true) +// +func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { + var err error + type exampleTag struct { + token string + linkText string + fileType string + } + exampleTags := []exampleTag{} + + // collect all example Tags + for _, mline := range mlines { + if mline.preformatted || !mline.beginTag { + continue + } + line := mline.data + if !strings.HasPrefix(line, exampleLineStart) { + continue + } + match := exampleMungeTagRE.FindStringSubmatch(line) + if len(match) < 4 { + err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line) + return mlines, err + } + tag := exampleTag{ + token: exampleToken + " " + match[1], + linkText: match[1], + fileType: match[3], + } + exampleTags = append(exampleTags, tag) + } + // update all example Tags + for _, tag := range exampleTags { + ft := "" + if tag.fileType == "json" { + ft = "json" + } + if tag.fileType == "yaml" { + ft = "yaml" + } + example, err := exampleContent(filePath, tag.linkText, ft) + if err != nil { + return mlines, err + } + mlines, err = updateMacroBlock(mlines, tag.token, example) + if err != nil { + return mlines, err + } + } + return mlines, nil +} + +// exampleContent retrieves the content of the file at linkPath +func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { + repoRel, err := makeRepoRelative(linkPath, filePath) + if err != nil { + return nil, err + } + + fileRel, err := makeFileRelative(linkPath, filePath) + if err != nil { + return nil, err + } + + dat, err := ioutil.ReadFile(repoRel) + if err != nil { + return nil, err + } + + // remove leading and trailing spaces and newlines + trimmedFileContent := strings.TrimSpace(string(dat)) + content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) + out := getMungeLines(content) + return out, nil +} diff --git a/_tools/mungedocs/example_syncer_test.go b/_tools/mungedocs/example_syncer_test.go new file mode 100644 index 0000000000..3d51b28c9d --- /dev/null +++ b/_tools/mungedocs/example_syncer_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func Test_syncExamples(t *testing.T) { + var podExample = `apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +` + var textExample = `some text +` + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", + }, + { + "\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", + }, + { + "\n\n", + "\n\n```\n" + textExample + "```\n\n[Download example](testdata/example.txt?raw=true)\n\n", + }, + } + repoRoot = "" + for _, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := syncExamples("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String()) + } + } +} diff --git a/_tools/mungedocs/headers.go b/_tools/mungedocs/headers.go new file mode 100644 index 0000000000..e23ae7536e --- /dev/null +++ b/_tools/mungedocs/headers.go @@ -0,0 +1,74 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" +) + +var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`) + +func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines { + var out mungeLines + + mline := mlines[linenum] + line := mlines[linenum].data + + matches := headerRegex.FindStringSubmatch(line) + if matches == nil { + out = append(out, mline) + return out + } + + // There must be a blank line before the # (unless first line in file) + if linenum != 0 { + newlen := len(newlines) + if newlines[newlen-1].data != "" { + out = append(out, blankMungeLine) + } + } + + // There must be a space AFTER the ##'s + newline := fmt.Sprintf("%s %s", matches[1], matches[2]) + newmline := newMungeLine(newline) + out = append(out, newmline) + + // The next line needs to be a blank line (unless last line in file) + if len(mlines) > linenum+1 && mlines[linenum+1].data != "" { + out = append(out, blankMungeLine) + } + return out +} + +// Header lines need whitespace around them and after the #s. +func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + for i, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + if !mline.header { + out = append(out, mline) + continue + } + newLines := fixHeaderLine(mlines, out, i) + out = append(out, newLines...) + } + return out, nil +} diff --git a/_tools/mungedocs/headers_test.go b/_tools/mungedocs/headers_test.go new file mode 100644 index 0000000000..d308b01034 --- /dev/null +++ b/_tools/mungedocs/headers_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestHeaderLines(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "# ok", + "# ok", + }, + { + "## ok", + "## ok", + }, + { + "##### ok", + "##### ok", + }, + { + "##fix", + "## fix", + }, + { + "foo\n\n##fix\n\nbar", + "foo\n\n## fix\n\nbar", + }, + { + "foo\n##fix\nbar", + "foo\n\n## fix\n\nbar", + }, + { + "foo\n```\n##fix\n```\nbar", + "foo\n```\n##fix\n```\nbar", + }, + { + "foo\n#fix1\n##fix2\nbar", + "foo\n\n# fix1\n\n## fix2\n\nbar", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateHeaderLines("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} diff --git a/_tools/mungedocs/kubectl_dash_f.go b/_tools/mungedocs/kubectl_dash_f.go new file mode 100644 index 0000000000..2dae48cc85 --- /dev/null +++ b/_tools/mungedocs/kubectl_dash_f.go @@ -0,0 +1,125 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path" + "strings" +) + +// Looks for lines that have kubectl commands with -f flags and files that +// don't exist. +func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) { + var errors []string + for i, mline := range mlines { + if !mline.preformatted { + continue + } + if err := lookForKubectl(mline.data, i); err != nil { + errors = append(errors, err.Error()) + } + } + err := error(nil) + if len(errors) != 0 { + err = fmt.Errorf("%s", strings.Join(errors, "\n")) + } + return mlines, err +} + +func lookForKubectl(line string, lineNum int) error { + fields := strings.Fields(line) + for i := range fields { + if fields[i] == "kubectl" { + return gotKubectl(lineNum, fields, i) + } + } + return nil +} + +func gotKubectl(lineNum int, fields []string, fieldNum int) error { + for i := fieldNum + 1; i < len(fields); i++ { + switch fields[i] { + case "create", "update", "replace", "delete": + return gotCommand(lineNum, fields, i) + } + } + return nil +} + +func gotCommand(lineNum int, fields []string, fieldNum int) error { + for i := fieldNum + 1; i < len(fields); i++ { + if strings.HasPrefix(fields[i], "-f") { + return gotDashF(lineNum, fields, i) + } + } + return nil +} + +func gotDashF(lineNum int, fields []string, fieldNum int) error { + target := "" + if fields[fieldNum] == "-f" { + if fieldNum+1 == len(fields) { + return fmt.Errorf("ran out of fields after '-f'") + } + target = fields[fieldNum+1] + } else { + target = fields[fieldNum][2:] + } + // Turn dirs into file-like names. + target = strings.TrimRight(target, "/") + + // Now exclude special-cases + + if target == "-" || target == "FILENAME" { + // stdin and "FILENAME" are OK + return nil + } + if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { + // URLs are ok + return nil + } + if strings.HasPrefix(target, "./") { + // Same-dir files are usually created in the same example + return nil + } + if strings.HasPrefix(target, "~/") { + // Home directory may also be created by the same example + return nil + } + if strings.HasPrefix(target, "/") { + // Absolute paths tend to be /tmp/* and created in the same example. + return nil + } + if strings.HasPrefix(target, "$") { + // Allow the start of the target to be an environment + // variable that points to the root of the kubernetes + // repo. + split := strings.SplitN(target, "/", 2) + if len(split) == 2 { + target = split[1] + } + } + + // If we got here we expect the file to exist. + _, err := os.Stat(path.Join(repoRoot, target)) + if os.IsNotExist(err) { + return fmt.Errorf("%d: target file %q does not exist", lineNum, target) + } + return err +} diff --git a/_tools/mungedocs/kubectl_dash_f_test.go b/_tools/mungedocs/kubectl_dash_f_test.go new file mode 100644 index 0000000000..6f18fd547a --- /dev/null +++ b/_tools/mungedocs/kubectl_dash_f_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import "testing" + +func TestKubectlDashF(t *testing.T) { + var cases = []struct { + in string + ok bool + }{ + // No match + {"", true}, + { + "Foo\nBar\n", + true, + }, + { + "Foo\nkubectl blah blech\nBar", + true, + }, + { + "Foo\n```shell\nkubectl blah blech\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create blech\n```\nBar", + true, + }, + // Special cases + { + "Foo\n```\nkubectl -blah create -f -\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f-\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f FILENAME\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fFILENAME\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f http://google.com/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fhttp://google.com/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f ./foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f./foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f /foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f~/foobar\n```\nBar", + true, + }, + // Real checks + { + "Foo\n```\nkubectl -blah create -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah update -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah update -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah replace -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah replace -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah delete -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah delete -fmungedocs.go\n```\nBar", + true, + }, + // Failures + { + "Foo\n```\nkubectl -blah delete -f does_not_exist\n```\nBar", + false, + }, + { + "Foo\n```\nkubectl -blah delete -fdoes_not_exist\n```\nBar", + false, + }, + } + for i, c := range cases { + repoRoot = "" + in := getMungeLines(c.in) + _, err := updateKubectlFileTargets("filename.md", in) + if err != nil && c.ok { + t.Errorf("case[%d]: expected success, got %v", i, err) + } + if err == nil && !c.ok { + t.Errorf("case[%d]: unexpected success", i) + } + } +} diff --git a/_tools/mungedocs/links.go b/_tools/mungedocs/links.go new file mode 100644 index 0000000000..c05cdfa187 --- /dev/null +++ b/_tools/mungedocs/links.go @@ -0,0 +1,238 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "net/url" + "os" + "path" + "regexp" + "strings" +) + +var ( + // Finds markdown links of the form [foo](bar "alt-text"). + linkRE = regexp.MustCompile(`\[([^]]*)\]\(([^)]*)\)`) + // Finds markdown link typos of the form (foo)[bar] + badLinkRE = regexp.MustCompile(`\([^]()]*\)\[[^]()]*\]`) + // Splits the link target into link target and alt-text. + altTextRE = regexp.MustCompile(`([^)]*)( ".*")`) +) + +func processLink(in string, filePath string) (string, error) { + var errs []string + out := linkRE.ReplaceAllStringFunc(in, func(in string) string { + var err error + match := linkRE.FindStringSubmatch(in) + if match == nil { + errs = append(errs, fmt.Sprintf("Detected this line had a link, but unable to parse, %v", in)) + return "" + } + // match[0] is the entire expression; + visibleText := match[1] + linkText := match[2] + altText := "" + if parts := altTextRE.FindStringSubmatch(linkText); parts != nil { + linkText = parts[1] + altText = parts[2] + } + + // clean up some random garbage I found in our docs. + linkText = strings.Trim(linkText, " ") + linkText = strings.Trim(linkText, "\n") + linkText = strings.Trim(linkText, " ") + + u, terr := url.Parse(linkText) + if terr != nil { + errs = append(errs, fmt.Sprintf("link %q is unparsable: %v", linkText, terr)) + return in + } + + if u.Host != "" && u.Host != "github.com" { + // We only care about relative links and links within github. + return in + } + + suggestedVisibleText := visibleText + if u.Path != "" && !strings.HasPrefix(linkText, "TODO:") { + newPath, targetExists := checkPath(filePath, path.Clean(u.Path)) + if !targetExists { + errs = append(errs, fmt.Sprintf("%q: target not found", linkText)) + return in + } + u.Path = newPath + if strings.HasPrefix(u.Path, "/") { + u.Host = "github.com" + u.Scheme = "https" + } else { + // Remove host and scheme from relative paths + u.Host = "" + u.Scheme = "" + } + // Make the visible text show the absolute path if it's + // not nested in or beneath the current directory. + if strings.HasPrefix(u.Path, "..") { + dir := path.Dir(filePath) + suggestedVisibleText, err = makeRepoRelative(path.Join(dir, u.Path), filePath) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: unable to make path relative", filePath)) + return in + } + } else { + suggestedVisibleText = u.Path + } + var unescaped string + if unescaped, err = url.QueryUnescape(u.String()); err != nil { + // Remove %28 type stuff, be nice to humans. + // And don't fight with the toc generator. + linkText = unescaped + } else { + linkText = u.String() + } + } + // If the current visible text is trying to be a file name, use + // the correct file name. + if strings.HasSuffix(visibleText, ".md") && !strings.ContainsAny(visibleText, ` '"`+"`") { + visibleText = suggestedVisibleText + } + + return fmt.Sprintf("[%s](%s)", visibleText, linkText+altText) + }) + if len(errs) != 0 { + return "", errors.New(strings.Join(errs, ",")) + } + return out, nil +} + +// updateLinks assumes lines has links in markdown syntax, and verifies that +// any relative links actually point to files that exist. +func updateLinks(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + allErrs := []string{} + + for lineNum, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + if badMatch := badLinkRE.FindString(mline.data); badMatch != "" { + allErrs = append(allErrs, + fmt.Sprintf("On line %d: found backwards markdown link %q", lineNum, badMatch)) + } + if !mline.link { + out = append(out, mline) + continue + } + line, err := processLink(mline.data, filePath) + if err != nil { + var s = fmt.Sprintf("On line %d: %s", lineNum, err.Error()) + err := errors.New(s) + allErrs = append(allErrs, err.Error()) + } + ml := newMungeLine(line) + out = append(out, ml) + } + err := error(nil) + if len(allErrs) != 0 { + err = fmt.Errorf("%s", strings.Join(allErrs, "\n")) + } + return out, err +} + +// We have to append together before path.Clean will be able to tell that stuff +// like ../docs isn't needed. +func cleanPath(dirPath, linkPath string) string { + clean := path.Clean(path.Join(dirPath, linkPath)) + if strings.HasPrefix(clean, dirPath+"/") { + out := strings.TrimPrefix(clean, dirPath+"/") + if out != linkPath { + fmt.Printf("%s -> %s\n", linkPath, out) + } + return out + } + return linkPath +} + +func checkPath(filePath, linkPath string) (newPath string, ok bool) { + dir := path.Dir(filePath) + absFilePrefixes := []string{ + "/kubernetes/kubernetes/blob/master/", + "/kubernetes/kubernetes/tree/master/", + } + for _, prefix := range absFilePrefixes { + if strings.HasPrefix(linkPath, prefix) { + linkPath = strings.TrimPrefix(linkPath, prefix) + // Now linkPath is relative to the root of the repo. The below + // loop that adds ../ at the beginning of the path should find + // the right path. + break + } + } + if strings.HasPrefix(linkPath, "/") { + // These links might go to e.g. the github issues page, or a + // file at a particular revision, or another github project + // entirely. + return linkPath, true + } + linkPath = cleanPath(dir, linkPath) + + // Fast exit if the link is already correct. + if info, err := os.Stat(path.Join(dir, linkPath)); err == nil { + if info.IsDir() { + return linkPath + "/", true + } + return linkPath, true + } + + for strings.HasPrefix(linkPath, "../") { + linkPath = strings.TrimPrefix(linkPath, "../") + } + + // Fix - vs _ automatically + nameMungers := []func(string) string{ + func(s string) string { return s }, + func(s string) string { return strings.Replace(s, "-", "_", -1) }, + func(s string) string { return strings.Replace(s, "_", "-", -1) }, + } + // Fix being moved into/out of admin (replace "admin" with directory + // you're doing mass movements to/from). + pathMungers := []func(string) string{ + func(s string) string { return s }, + func(s string) string { return path.Join("admin", s) }, + func(s string) string { return strings.TrimPrefix(s, "admin/") }, + } + + for _, namer := range nameMungers { + for _, pather := range pathMungers { + newPath = pather(namer(linkPath)) + for i := 0; i < 7; i++ { + // The file must exist. + target := path.Join(dir, newPath) + if info, err := os.Stat(target); err == nil { + if info.IsDir() { + return newPath + "/", true + } + return cleanPath(dir, newPath), true + } + newPath = path.Join("..", newPath) + } + } + } + return linkPath, false +} diff --git a/_tools/mungedocs/links_test.go b/_tools/mungedocs/links_test.go new file mode 100644 index 0000000000..37f8e53171 --- /dev/null +++ b/_tools/mungedocs/links_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "testing" +) + +var _ = fmt.Printf + +func TestBadLinks(t *testing.T) { + var cases = []struct { + in string + }{ + {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/NOTREADME.md)"}, + {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/docs/NOTREADME.md)"}, + {"[NOTREADME](../NOTREADME.md)"}, + } + for _, c := range cases { + in := getMungeLines(c.in) + _, err := updateLinks("filename.md", in) + if err == nil { + t.Errorf("Expected error") + } + } +} +func TestGoodLinks(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + {"[README](https://lwn.net)", + "[README](https://lwn.net)"}, + // _ to - + {"[README](https://github.com/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/test_dashes.md)", + "[README](../../cmd/mungedocs/testdata/test-dashes.md)"}, + // - to _ + {"[README](../../cmd/mungedocs/testdata/test-underscores.md)", + "[README](../../cmd/mungedocs/testdata/test_underscores.md)"}, + + // Does this even make sense? i dunno + {"[README](/docs/README.md)", + "[README](https://github.com/docs/README.md)"}, + {"[README](/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/README.md)", + "[README](../../cmd/mungedocs/testdata/README.md)"}, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateLinks("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} diff --git a/_tools/mungedocs/mungedocs.go b/_tools/mungedocs/mungedocs.go new file mode 100644 index 0000000000..ef3eb2da1a --- /dev/null +++ b/_tools/mungedocs/mungedocs.go @@ -0,0 +1,235 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" +) + +var ( + verbose = flag.Bool("verbose", false, "On verification failure, emit pre-munge and post-munge versions.") + verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") + norecurse = flag.Bool("norecurse", false, "Only process the files of --root-dir.") + upstream = flag.String("upstream", "upstream", "The name of the upstream Git remote to pull from") + rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") + // "repo-root" seems like a dumb name, this is the relative path (from rootDir) to get to the repoRoot + relRoot = flag.String("repo-root", "..", `Appended to --root-dir to get the repository root. +It's done this way so that generally you just have to set --root-dir. +Examples: + * --root-dir=docs/ --repo-root=.. means the repository root is ./ + * --root-dir=/usr/local/long/path/repo/docs/ --repo-root=.. means the repository root is /usr/local/long/path/repo/ + * --root-dir=/usr/local/long/path/repo/docs/admin --repo-root=../.. means the repository root is /usr/local/long/path/repo/`) + skipMunges = flag.String("skip-munges", "", "Comma-separated list of munges to *not* run. Available munges are: "+availableMungeList) + repoRoot string + + ErrChangesNeeded = errors.New("mungedocs: changes required") + + // All of the munge operations to perform. + // TODO: allow selection from command line. (e.g., just check links in the examples directory.) + allMunges = []munge{ + // Simple "check something" functions must run first. + {"preformat-balance", checkPreformatBalance}, + // Functions which modify state. + {"remove-whitespace", updateWhitespace}, + {"table-of-contents", updateTOC}, + {"md-links", updateLinks}, + {"blank-lines-surround-preformatted", updatePreformatted}, + {"header-lines", updateHeaderLines}, + {"analytics", updateAnalytics}, + {"kubectl-dash-f", updateKubectlFileTargets}, + {"sync-examples", syncExamples}, + } + availableMungeList = func() string { + names := []string{} + for _, m := range allMunges { + names = append(names, m.name) + } + return strings.Join(names, ",") + }() +) + +// a munge processes a document, returning an updated document xor an error. +// The fn is NOT allowed to mutate 'before', if changes are needed it must copy +// data into a new byte array and return that. +type munge struct { + name string + fn func(filePath string, mlines mungeLines) (after mungeLines, err error) +} + +type fileProcessor struct { + // Which munge functions should we call? + munges []munge + + // Are we allowed to make changes? + verifyOnly bool +} + +// Either change a file or verify that it needs no changes (according to modify argument) +func (f fileProcessor) visit(path string) error { + if !strings.HasSuffix(path, ".md") { + return nil + } + + fileBytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + mungeLines := getMungeLines(string(fileBytes)) + + modificationsMade := false + errFound := false + filePrinted := false + for _, munge := range f.munges { + after, err := munge.fn(path, mungeLines) + if err != nil || !after.Equal(mungeLines) { + if !filePrinted { + fmt.Printf("%s\n----\n", path) + filePrinted = true + } + fmt.Printf("%s:\n", munge.name) + if *verbose { + if len(mungeLines) <= 20 { + fmt.Printf("INPUT: <<<%v>>>\n", mungeLines) + fmt.Printf("MUNGED: <<<%v>>>\n", after) + } else { + fmt.Printf("not printing failed chunk: too many lines\n") + } + } + if err != nil { + fmt.Println(err) + errFound = true + } else { + fmt.Println("contents were modified") + modificationsMade = true + } + fmt.Println("") + } + mungeLines = after + } + + // Write out new file with any changes. + if modificationsMade { + if f.verifyOnly { + // We're not allowed to make changes. + return ErrChangesNeeded + } + ioutil.WriteFile(path, mungeLines.Bytes(), 0644) + } + if errFound { + return ErrChangesNeeded + } + + return nil +} + +func newWalkFunc(fp *fileProcessor, changesNeeded *bool) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + stat, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if path != *rootDir && stat.IsDir() && *norecurse { + return filepath.SkipDir + } + if err := fp.visit(path); err != nil { + *changesNeeded = true + if err != ErrChangesNeeded { + return err + } + } + return nil + } +} + +func wantedMunges() (filtered []munge) { + skipList := strings.Split(*skipMunges, ",") + skipped := map[string]bool{} + for _, m := range skipList { + if len(m) > 0 { + skipped[m] = true + } + } + for _, m := range allMunges { + if !skipped[m.name] { + filtered = append(filtered, m) + } else { + // Remove from the map so we can verify that everything + // requested was in fact valid. + delete(skipped, m.name) + } + } + if len(skipped) != 0 { + fmt.Fprintf(os.Stderr, "ERROR: requested to skip %v, but these are not valid munges. (valid: %v)\n", skipped, availableMungeList) + os.Exit(1) + } + return filtered +} + +func main() { + var err error + flag.Parse() + + if *rootDir == "" { + fmt.Fprintf(os.Stderr, "usage: %s [--help] [--verify] [--norecurse] --root-dir [--skip-munges=] [--upstream=] \n", flag.Arg(0)) + os.Exit(1) + } + + repoRoot = path.Join(*rootDir, *relRoot) + repoRoot, err = filepath.Abs(repoRoot) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(2) + } + + fp := fileProcessor{ + munges: wantedMunges(), + verifyOnly: *verify, + } + + // For each markdown file under source docs root, process the doc. + // - If any error occurs: exit with failure (exit >1). + // - If verify is true: exit 0 if no changes needed, exit 1 if changes + // needed. + // - If verify is false: exit 0 if changes successfully made or no + // changes needed, exit 1 if manual changes are needed. + var changesNeeded bool + + err = filepath.Walk(*rootDir, newWalkFunc(&fp, &changesNeeded)) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(2) + } + if changesNeeded { + if *verify { + fmt.Fprintf(os.Stderr, "FAIL: changes needed but not made due to --verify\n") + } else { + fmt.Fprintf(os.Stderr, "FAIL: some manual changes are still required.\n") + } + os.Exit(1) + } +} diff --git a/_tools/mungedocs/preformatted.go b/_tools/mungedocs/preformatted.go new file mode 100644 index 0000000000..582ba981a1 --- /dev/null +++ b/_tools/mungedocs/preformatted.go @@ -0,0 +1,51 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import "fmt" + +// Blocks of ``` need to have blank lines on both sides or they don't look +// right in HTML. +func updatePreformatted(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + inpreformat := false + for i, mline := range mlines { + if !inpreformat && mline.preformatted { + if i == 0 || out[len(out)-1].data != "" { + out = append(out, blankMungeLine) + } + // start of a preformat block + inpreformat = true + } + out = append(out, mline) + if inpreformat && !mline.preformatted { + if i >= len(mlines)-2 || mlines[i+1].data != "" { + out = append(out, blankMungeLine) + } + inpreformat = false + } + } + return out, nil +} + +// If the file ends on a preformatted line, there must have been an imbalance. +func checkPreformatBalance(filePath string, mlines mungeLines) (mungeLines, error) { + if len(mlines) > 0 && mlines[len(mlines)-1].preformatted { + return mlines, fmt.Errorf("unbalanced triple backtick delimiters") + } + return mlines, nil +} diff --git a/_tools/mungedocs/preformatted_test.go b/_tools/mungedocs/preformatted_test.go new file mode 100644 index 0000000000..ec93854ba0 --- /dev/null +++ b/_tools/mungedocs/preformatted_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestPreformatted(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "```\nbob\n```", + "\n```\nbob\n```\n\n", + }, + { + "```\nbob\n```\n```\nnotbob\n```\n", + "\n```\nbob\n```\n\n```\nnotbob\n```\n\n", + }, + { + "```bob```\n", + "```bob```\n", + }, + { + " ```\n bob\n ```", + "\n ```\n bob\n ```\n\n", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updatePreformatted("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} + +func TestPreformattedImbalance(t *testing.T) { + var cases = []struct { + in string + ok bool + }{ + {"", true}, + {"```\nin\n```", true}, + {"```\nin\n```\nout", true}, + {"```", false}, + {"```\nin\n```\nout\n```", false}, + } + for i, c := range cases { + in := getMungeLines(c.in) + out, err := checkPreformatBalance("filename.md", in) + if err != nil && c.ok { + t.Errorf("case[%d]: expected success", i) + } + if err == nil && !c.ok { + t.Errorf("case[%d]: expected failure", i) + } + // Even in case of misformat, return all the text, + // so that the user's work is not lost. + if !equalMungeLines(out, in) { + t.Errorf("case[%d]: expected munged text to be identical to input text", i) + } + } +} + +func equalMungeLines(a, b mungeLines) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/_tools/mungedocs/testdata/README.md b/_tools/mungedocs/testdata/README.md new file mode 100644 index 0000000000..7b57bd29ea --- /dev/null +++ b/_tools/mungedocs/testdata/README.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/example.txt b/_tools/mungedocs/testdata/example.txt new file mode 100644 index 0000000000..7b57bd29ea --- /dev/null +++ b/_tools/mungedocs/testdata/example.txt @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/pod.yaml b/_tools/mungedocs/testdata/pod.yaml new file mode 100644 index 0000000000..89920b83a9 --- /dev/null +++ b/_tools/mungedocs/testdata/pod.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 \ No newline at end of file diff --git a/_tools/mungedocs/testdata/test-dashes.md b/_tools/mungedocs/testdata/test-dashes.md new file mode 100644 index 0000000000..7b57bd29ea --- /dev/null +++ b/_tools/mungedocs/testdata/test-dashes.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/test_underscores.md b/_tools/mungedocs/testdata/test_underscores.md new file mode 100644 index 0000000000..7b57bd29ea --- /dev/null +++ b/_tools/mungedocs/testdata/test_underscores.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/toc.go b/_tools/mungedocs/toc.go new file mode 100644 index 0000000000..649a954bf5 --- /dev/null +++ b/_tools/mungedocs/toc.go @@ -0,0 +1,89 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" + "strings" +) + +const tocMungeTag = "GENERATED_TOC" + +var r = regexp.MustCompile("[^A-Za-z0-9-]") + +// inserts/updates a table of contents in markdown file. +// +// First, builds a ToC. +// Then, finds the magic macro block tags and replaces anything between those with +// the ToC, thereby updating any previously inserted ToC. +// +// TODO(erictune): put this in own package with tests +func updateTOC(filePath string, mlines mungeLines) (mungeLines, error) { + toc := buildTOC(mlines) + updatedMarkdown, err := updateMacroBlock(mlines, tocMungeTag, toc) + if err != nil { + return mlines, err + } + return updatedMarkdown, nil +} + +// builds table of contents for markdown file +// +// First scans for all section headers (lines that begin with "#" but not within code quotes) +// and builds a table of contents from those. Assumes bookmarks for those will be +// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. +// builds the ToC. + +func buildTOC(mlines mungeLines) mungeLines { + var out mungeLines + bookmarks := map[string]int{} + + for _, mline := range mlines { + if mline.preformatted || !mline.header { + continue + } + // Add a blank line after the munge start tag + if len(out) == 0 { + out = append(out, blankMungeLine) + } + line := mline.data + noSharps := strings.TrimLeft(line, "#") + numSharps := len(line) - len(noSharps) + heading := strings.Trim(noSharps, " \n") + if numSharps > 0 { + indent := strings.Repeat(" ", numSharps-1) + bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) + // remove symbols (except for -) in bookmarks + bookmark = r.ReplaceAllString(bookmark, "") + // Incremental counter for duplicate bookmarks + next := bookmarks[bookmark] + bookmarks[bookmark] = next + 1 + if next > 0 { + bookmark = fmt.Sprintf("%s-%d", bookmark, next) + } + tocLine := fmt.Sprintf("%s- [%s](#%s)", indent, heading, bookmark) + out = append(out, newMungeLine(tocLine)) + } + + } + // Add a blank line before the munge end tag + if len(out) != 0 { + out = append(out, blankMungeLine) + } + return out +} diff --git a/_tools/mungedocs/toc_test.go b/_tools/mungedocs/toc_test.go new file mode 100644 index 0000000000..95ff354ed5 --- /dev/null +++ b/_tools/mungedocs/toc_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func Test_buildTOC(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + {"Lorem ipsum\ndolor sit amet\n", ""}, + { + "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", + }, + { + "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n```bash\n#!/bin/sh\n```", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", + }, + { + "# Title\nLorem ipsum \n## Section Heading\n### Ok, why doesn't this work? ...add 4 *more* `symbols`!\ndolor sit amet\n", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n - [Ok, why doesn't this work? ...add 4 *more* `symbols`!](#ok-why-doesnt-this-work-add-4-more-symbols)\n\n", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual := buildTOC(in) + if !expected.Equal(actual) { + t.Errorf("Case[%d] Expected TOC '%v' but got '%v'", i, expected.String(), actual.String()) + } + } +} + +func Test_updateTOC(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "Lorem ipsum\ndolor sit amet\n", + "Lorem ipsum\ndolor sit amet\n", + }, + { + "# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", + "# Title\nLorem ipsum \n**table of contents**\n\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n", + }, + } + for _, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateTOC("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + if !expected.Equal(actual) { + t.Errorf("Expected TOC '%v' but got '%v'", expected.String(), actual.String()) + } + } +} diff --git a/_tools/mungedocs/util.go b/_tools/mungedocs/util.go new file mode 100644 index 0000000000..c25e1d1976 --- /dev/null +++ b/_tools/mungedocs/util.go @@ -0,0 +1,291 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "path" + "path/filepath" + "regexp" + "strings" + "unicode" +) + +// Replaces the text between matching "beginMark" and "endMark" within the +// document represented by "lines" with "insertThis". +// +// Delimiters should occupy own line. +// Returns copy of document with modifications. +func updateMacroBlock(mlines mungeLines, token string, insertThis mungeLines) (mungeLines, error) { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + var out mungeLines + betweenBeginAndEnd := false + for _, mline := range mlines { + if mline.preformatted && !betweenBeginAndEnd { + out = append(out, mline) + continue + } + line := mline.data + if mline.beginTag && line == beginMark { + if betweenBeginAndEnd { + return nil, fmt.Errorf("found second begin mark while updating macro blocks") + } + betweenBeginAndEnd = true + out = append(out, mline) + } else if mline.endTag && line == endMark { + if !betweenBeginAndEnd { + return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") + } + betweenBeginAndEnd = false + out = append(out, insertThis...) + out = append(out, mline) + } else { + if !betweenBeginAndEnd { + out = append(out, mline) + } + } + } + if betweenBeginAndEnd { + return nil, fmt.Errorf("never found closing end mark while updating macro blocks") + } + return out, nil +} + +// Tests that a document, represented as a slice of lines, has a line. Ignores +// leading and trailing space. +func hasLine(lines mungeLines, needle string) bool { + for _, mline := range lines { + haystack := strings.TrimSpace(mline.data) + if haystack == needle { + return true + } + } + return false +} + +func removeMacroBlock(token string, mlines mungeLines) (mungeLines, error) { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + var out mungeLines + betweenBeginAndEnd := false + for _, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + line := mline.data + if mline.beginTag && line == beginMark { + if betweenBeginAndEnd { + return nil, fmt.Errorf("found second begin mark while updating macro blocks") + } + betweenBeginAndEnd = true + } else if mline.endTag && line == endMark { + if !betweenBeginAndEnd { + return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") + } + betweenBeginAndEnd = false + } else { + if !betweenBeginAndEnd { + out = append(out, mline) + } + } + } + if betweenBeginAndEnd { + return nil, fmt.Errorf("never found closing end mark while updating macro blocks") + } + return out, nil +} + +// Add a macro block to the beginning of a set of lines +func prependMacroBlock(token string, mlines mungeLines) mungeLines { + beginLine := newMungeLine(beginMungeTag(token)) + endLine := newMungeLine(endMungeTag(token)) + out := mungeLines{beginLine, endLine} + if len(mlines) > 0 && mlines[0].data != "" { + out = append(out, blankMungeLine) + } + return append(out, mlines...) +} + +// Add a macro block to the end of a set of lines +func appendMacroBlock(mlines mungeLines, token string) mungeLines { + beginLine := newMungeLine(beginMungeTag(token)) + endLine := newMungeLine(endMungeTag(token)) + out := mlines + if len(mlines) > 0 && mlines[len(mlines)-1].data != "" { + out = append(out, blankMungeLine) + } + return append(out, beginLine, endLine) +} + +// Tests that a document, represented as a slice of lines, has a macro block. +func hasMacroBlock(lines mungeLines, token string) bool { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + + foundBegin := false + for _, mline := range lines { + if mline.preformatted { + continue + } + if !mline.beginTag && !mline.endTag { + continue + } + line := mline.data + switch { + case !foundBegin && line == beginMark: + foundBegin = true + case foundBegin && line == endMark: + return true + } + } + return false +} + +// Returns the canonical begin-tag for a given description. This does not +// include the trailing newline. +func beginMungeTag(desc string) string { + return fmt.Sprintf("", desc) +} + +// Returns the canonical end-tag for a given description. This does not +// include the trailing newline. +func endMungeTag(desc string) string { + return fmt.Sprintf("", desc) +} + +type mungeLine struct { + data string + preformatted bool + header bool + link bool + beginTag bool + endTag bool +} + +type mungeLines []mungeLine + +func (m1 mungeLines) Equal(m2 mungeLines) bool { + if len(m1) != len(m2) { + return false + } + for i := range m1 { + if m1[i].data != m2[i].data { + return false + } + } + return true +} + +func (mlines mungeLines) String() string { + slice := []string{} + for _, mline := range mlines { + slice = append(slice, mline.data) + } + s := strings.Join(slice, "\n") + // We need to tack on an extra newline at the end of the file + return s + "\n" +} + +func (mlines mungeLines) Bytes() []byte { + return []byte(mlines.String()) +} + +var ( + // Finds all preformatted block start/stops. + preformatRE = regexp.MustCompile("^\\s*```") + notPreformatRE = regexp.MustCompile("^\\s*```.*```") + // Is this line a header? + mlHeaderRE = regexp.MustCompile(`^#`) + // Is there a link on this line? + mlLinkRE = regexp.MustCompile(`\[[^]]*\]\([^)]*\)`) + beginTagRE = regexp.MustCompile(`