From b8411c9084307a4e2914b28a9052fee28ec595a0 Mon Sep 17 00:00:00 2001 From: Anthony Date: Sat, 19 Aug 2017 19:02:24 +0800 Subject: [PATCH 01/27] Update daemonset.md modify "nodes ips" to "node IPs" --- docs/concepts/workloads/controllers/daemonset.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts/workloads/controllers/daemonset.md b/docs/concepts/workloads/controllers/daemonset.md index 3e06d892c6d2c..5b26af426c137 100644 --- a/docs/concepts/workloads/controllers/daemonset.md +++ b/docs/concepts/workloads/controllers/daemonset.md @@ -112,7 +112,7 @@ Some possible patterns for communicating with pods in a DaemonSet are: - **Push**: Pods in the DaemonSet are configured to send updates to another service, such as a stats database. They do not have clients. -- **NodeIP and Known Port**: Pods in the DaemonSet can use a `hostPort`, so that the pods are reachable via the node IPs. Clients know the list of nodes ips somehow, and know the port by convention. +- **NodeIP and Known Port**: Pods in the DaemonSet can use a `hostPort`, so that the pods are reachable via the node IPs. Clients know the list of node IPs somehow, and know the port by convention. - **DNS**: Create a [headless service](/docs/user-guide/services/#headless-services) with the same pod selector, and then discover DaemonSets using the `endpoints` resource or retrieve multiple A records from DNS. From 9d56d650cf701d264fac8a7f420e750d0ba4fc52 Mon Sep 17 00:00:00 2001 From: Stewart-YU Date: Tue, 22 Aug 2017 19:54:07 +0800 Subject: [PATCH 02/27] Update update-daemon-set.md Fix a TODO url. --- docs/tasks/manage-daemon/update-daemon-set.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tasks/manage-daemon/update-daemon-set.md b/docs/tasks/manage-daemon/update-daemon-set.md index ffdb08d921a6d..8a035bf6fd8df 100644 --- a/docs/tasks/manage-daemon/update-daemon-set.md +++ b/docs/tasks/manage-daemon/update-daemon-set.md @@ -198,7 +198,7 @@ progress. * See [Task: Performing a rollback on a DaemonSet](/docs/tasks/manage-daemon/rollback-daemon-set/) -* *TODO: Link to "Task: Creating a DaemonSet to adopt existing DaemonSet pods"* +* See [Concepts: Creating a DaemonSet to adopt existing DaemonSet pods](/docs/concepts/workloads/controllers/daemonset/) {% endcapture %} From b8bb7f0c3d3c73585d481017ac3190d7b7523a08 Mon Sep 17 00:00:00 2001 From: Quentin Revel Date: Tue, 22 Aug 2017 15:14:16 +0200 Subject: [PATCH 03/27] Update basic-stateful-set.md --- docs/tutorials/stateful-application/basic-stateful-set.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/stateful-application/basic-stateful-set.md b/docs/tutorials/stateful-application/basic-stateful-set.md index afa6d06c6d106..f8ea44238a753 100644 --- a/docs/tutorials/stateful-application/basic-stateful-set.md +++ b/docs/tutorials/stateful-application/basic-stateful-set.md @@ -667,7 +667,7 @@ web-2 1/1 Running 0 18s Get the Pod's container. ```shell{% raw %} -get po web-2 --template '{{range $i, $c := .spec.containers}}{{$c.image}}{{end}}' +kubectl get po web-2 --template '{{range $i, $c := .spec.containers}}{{$c.image}}{{end}}' gcr.io/google_containers/nginx-slim:0.8 {% endraw %} ``` From 6961877c8bf8a582dedadce2f924dbdca0bbf172 Mon Sep 17 00:00:00 2001 From: Quentin Revel Date: Tue, 22 Aug 2017 15:21:31 +0200 Subject: [PATCH 04/27] Update basic-stateful-set.md --- docs/tutorials/stateful-application/basic-stateful-set.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/stateful-application/basic-stateful-set.md b/docs/tutorials/stateful-application/basic-stateful-set.md index f8ea44238a753..30c13a4f8e9da 100644 --- a/docs/tutorials/stateful-application/basic-stateful-set.md +++ b/docs/tutorials/stateful-application/basic-stateful-set.md @@ -738,7 +738,7 @@ web-1 1/1 Running 0 18s Get the `web-1` Pods container. ```shell{% raw %} -get po web-1 --template '{{range $i, $c := .spec.containers}}{{$c.image}}{{end}}' +kubectl get po web-1 --template '{{range $i, $c := .spec.containers}}{{$c.image}}{{end}}' gcr.io/google_containers/nginx-slim:0.8 {% endraw %} ``` From fc453318a9ca7bdc4387450ca43485b5a7ce5d9f Mon Sep 17 00:00:00 2001 From: XsWack Date: Tue, 22 Aug 2017 22:02:09 +0800 Subject: [PATCH 05/27] Update update-daemon-set.md The output is wrong and it should be "daemonset" --- docs/tasks/manage-daemon/update-daemon-set.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tasks/manage-daemon/update-daemon-set.md b/docs/tasks/manage-daemon/update-daemon-set.md index ffdb08d921a6d..870315e896e14 100644 --- a/docs/tasks/manage-daemon/update-daemon-set.md +++ b/docs/tasks/manage-daemon/update-daemon-set.md @@ -145,7 +145,7 @@ kubectl rollout status ds/ When the rollout is complete, the output is similar to this: ```shell -daemon set "" successfully rolled out +daemonset "" successfully rolled out ``` ## Troubleshooting From ca31adc78c2d2958f9f80eeba90259f9924991f9 Mon Sep 17 00:00:00 2001 From: Daniel Nardo Date: Wed, 5 Jul 2017 13:02:30 -0700 Subject: [PATCH 06/27] Update ip-masq-agent docs: - adds the node label requirements to the steps for running ip-masq-agent - change the wording on when this is the default. --- docs/tasks/administer-cluster/ip-masq-agent.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/tasks/administer-cluster/ip-masq-agent.md b/docs/tasks/administer-cluster/ip-masq-agent.md index 9ed4987e48f52..325a8c6a1ad6f 100644 --- a/docs/tasks/administer-cluster/ip-masq-agent.md +++ b/docs/tasks/administer-cluster/ip-masq-agent.md @@ -50,7 +50,7 @@ MASQUERADE all -- anywhere anywhere /* ip-masq-agent: ``` -By default, in GCE/GKE starting with Kubernetes version 1.7.0, the ip-masq-agent will run in your cluster. If you are running in another environment, you can add the ip-masq-agent [DaemonSet](/docs/concepts/workloads/controllers/daemonset/) to your cluster: +By default, in GCE/GKE starting with Kubernetes version 1.7.0, if network policy is enabled or you are using a cluster CIDR not in the 10.0.0.0/8 range, the ip-masq-agent will run in your cluster. If you are running in another environment, you can add the ip-masq-agent [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) to your cluster: {% endcapture %} @@ -63,6 +63,12 @@ To create an ip-masq-agent, run the following kubectl command: kubectl create -f https://raw.githubusercontent.com/kubernetes-incubator/ip-masq-agent/master/ip-masq-agent.yaml ` +You must also apply the appropriate node label to any nodes in your cluster that you want the agent to run on. + +` +kubectl label nodes my-node beta.kubernetes.io/masq-agent-ds-ready=true +` + More information can be found in the ip-masq-agent documentation [here](https://github.com/kubernetes-incubator/ip-masq-agent) In most cases, the default set of rules should be sufficient; however, if this is not the case for your cluster, you can create and apply a [ConfigMap](/docs/tasks/configure-pod-container/configmap/) to customize the IP ranges that are affected. For example, to allow only 10.0.0.0/8 to be considered by the ip-masq-agent, you can create the following [ConfigMap](/docs/tasks/configure-pod-container/configmap/) in a file called "config". From 1245055dfcfe951060e65c5b3d7f0755e3bf2475 Mon Sep 17 00:00:00 2001 From: XsWack Date: Thu, 24 Aug 2017 04:13:52 +0800 Subject: [PATCH 07/27] Update managing-tls-in-a-cluster.md (#5101) fix bad url --- docs/tasks/tls/managing-tls-in-a-cluster.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tasks/tls/managing-tls-in-a-cluster.md b/docs/tasks/tls/managing-tls-in-a-cluster.md index 308613f589c13..30299af8c473e 100644 --- a/docs/tasks/tls/managing-tls-in-a-cluster.md +++ b/docs/tasks/tls/managing-tls-in-a-cluster.md @@ -111,7 +111,7 @@ Notice that the `server.csr` file created in step 1 is base64 encoded and stashed in the `.spec.request` field. We are also requesting a certificate with the "digital signature", "key encipherment", and "server auth" key usages. We support all key usages and extended key usages listed -[here](https://godoc.org/k8s.io/client-go/pkg/apis/certificates/v1beta1#KeyUsage) +[here](https://godoc.org/k8s.io/api/certificates/v1beta1#KeyUsage) so you can request client certificates and other certificates using this same API. From 618c212c32f792e60becb7b3168e13a537135315 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 24 Aug 2017 09:36:36 -0700 Subject: [PATCH 08/27] Move mungedocs tool to the docs repo. (#5178) * Remove seemingly errant file * Move mungedocs tool to docs repo * Bump go version to 1.8.3 --- .travis.yml | 6 +- _tools/mungedocs/README.md | 22 ++ _tools/mungedocs/analytics.go | 58 ++++ _tools/mungedocs/analytics_test.go | 94 ++++++ _tools/mungedocs/example_syncer.go | 121 ++++++++ _tools/mungedocs/example_syncer_test.go | 68 ++++ _tools/mungedocs/headers.go | 74 +++++ _tools/mungedocs/headers_test.go | 74 +++++ _tools/mungedocs/kubectl_dash_f.go | 125 ++++++++ _tools/mungedocs/kubectl_dash_f_test.go | 143 +++++++++ _tools/mungedocs/links.go | 238 ++++++++++++++ _tools/mungedocs/links_test.go | 75 +++++ _tools/mungedocs/mungedocs.go | 235 ++++++++++++++ _tools/mungedocs/preformatted.go | 51 +++ _tools/mungedocs/preformatted_test.go | 98 ++++++ _tools/mungedocs/testdata/README.md | 1 + _tools/mungedocs/testdata/example.txt | 1 + _tools/mungedocs/testdata/pod.yaml | 10 + _tools/mungedocs/testdata/test-dashes.md | 1 + _tools/mungedocs/testdata/test_underscores.md | 1 + _tools/mungedocs/toc.go | 89 ++++++ _tools/mungedocs/toc_test.go | 79 +++++ _tools/mungedocs/util.go | 291 ++++++++++++++++++ _tools/mungedocs/util_test.go | 172 +++++++++++ _tools/mungedocs/whitespace.go | 31 ++ _tools/mungedocs/whitespace_test.go | 46 +++ quota-pod-deployment.yaml | 14 - 27 files changed, 2201 insertions(+), 17 deletions(-) create mode 100644 _tools/mungedocs/README.md create mode 100644 _tools/mungedocs/analytics.go create mode 100644 _tools/mungedocs/analytics_test.go create mode 100644 _tools/mungedocs/example_syncer.go create mode 100644 _tools/mungedocs/example_syncer_test.go create mode 100644 _tools/mungedocs/headers.go create mode 100644 _tools/mungedocs/headers_test.go create mode 100644 _tools/mungedocs/kubectl_dash_f.go create mode 100644 _tools/mungedocs/kubectl_dash_f_test.go create mode 100644 _tools/mungedocs/links.go create mode 100644 _tools/mungedocs/links_test.go create mode 100644 _tools/mungedocs/mungedocs.go create mode 100644 _tools/mungedocs/preformatted.go create mode 100644 _tools/mungedocs/preformatted_test.go create mode 100644 _tools/mungedocs/testdata/README.md create mode 100644 _tools/mungedocs/testdata/example.txt create mode 100644 _tools/mungedocs/testdata/pod.yaml create mode 100644 _tools/mungedocs/testdata/test-dashes.md create mode 100644 _tools/mungedocs/testdata/test_underscores.md create mode 100644 _tools/mungedocs/toc.go create mode 100644 _tools/mungedocs/toc_test.go create mode 100644 _tools/mungedocs/util.go create mode 100644 _tools/mungedocs/util_test.go create mode 100644 _tools/mungedocs/whitespace.go create mode 100644 _tools/mungedocs/whitespace_test.go delete mode 100644 quota-pod-deployment.yaml diff --git a/.travis.yml b/.travis.yml index 6d889f3c237ab..964b903136535 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.8.1 + - 1.8.3 # Don't want default ./... here: install: @@ -19,8 +19,8 @@ install: - git clone --depth=50 --branch=master https://github.com/kubernetes/md-check $HOME/gopath/src/k8s.io/md-check - go get -t -v k8s.io/md-check -# (3) Fetch mungedocs -- go get -v k8s.io/kubernetes/cmd/mungedocs +# (3) Build mungedocs +- go install ./_tools/mungedocs script: - go test -v k8s.io/kubernetes.github.io/test diff --git a/_tools/mungedocs/README.md b/_tools/mungedocs/README.md new file mode 100644 index 0000000000000..5fe3ed106cf45 --- /dev/null +++ b/_tools/mungedocs/README.md @@ -0,0 +1,22 @@ +# Documentation Mungers + +Basically this is like lint/gofmt for md docs. + +It basically does the following: +- iterate over all files in the given doc root. +- for each file split it into a slice (mungeLines) of lines (mungeLine) +- a mungeline has metadata about each line typically determined by a 'fast' regex. + - metadata contains things like 'is inside a preformatted block' + - contains a markdown header + - has a link to another file + - etc.. + - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one. +- each munger is then called in turn + - they are given the mungeLines + - they create an entirely new set of mungeLines with their modifications + - the new set is returned +- the new set is then fed into the next munger. +- in the end we might commit the end mungeLines to the file or not (--verify) + + +[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]() diff --git a/_tools/mungedocs/analytics.go b/_tools/mungedocs/analytics.go new file mode 100644 index 0000000000000..a7eaefa0803d4 --- /dev/null +++ b/_tools/mungedocs/analytics.go @@ -0,0 +1,58 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "strings" +) + +const analyticsMungeTag = "GENERATED_ANALYTICS" +const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/" + +func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + fileName, err := makeRepoRelative(fileName, fileName) + if err != nil { + return mlines, err + } + + link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName) + insertLines := getMungeLines(link) + mlines, err = removeMacroBlock(analyticsMungeTag, mlines) + if err != nil { + return mlines, err + } + + // Remove floating analytics links not surrounded by the munge tags. + for _, mline := range mlines { + if mline.preformatted || mline.header || mline.beginTag || mline.endTag { + out = append(out, mline) + continue + } + if strings.HasPrefix(mline.data, analyticsLinePrefix) { + continue + } + out = append(out, mline) + } + out = appendMacroBlock(out, analyticsMungeTag) + out, err = updateMacroBlock(out, analyticsMungeTag, insertLines) + if err != nil { + return mlines, err + } + return out, nil +} diff --git a/_tools/mungedocs/analytics_test.go b/_tools/mungedocs/analytics_test.go new file mode 100644 index 0000000000000..99306f2fe00e3 --- /dev/null +++ b/_tools/mungedocs/analytics_test.go @@ -0,0 +1,94 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestAnalytics(t *testing.T) { + b := beginMungeTag("GENERATED_ANALYTICS") + e := endMungeTag("GENERATED_ANALYTICS") + var cases = []struct { + in string + expected string + }{ + { + "aoeu", + "aoeu" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()", + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "prefix" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + + "\n" + "suffix", + "prefix" + "\n" + "suffix" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + { + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n", + "aoeu" + "\n" + "\n" + "\n" + + b + "\n" + + "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + + e + "\n"}, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + out, err := updateAnalytics("path/to/file-name.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !expected.Equal(out) { + t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String()) + } + } +} diff --git a/_tools/mungedocs/example_syncer.go b/_tools/mungedocs/example_syncer.go new file mode 100644 index 0000000000000..c15255be43948 --- /dev/null +++ b/_tools/mungedocs/example_syncer.go @@ -0,0 +1,121 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "io/ioutil" + "regexp" + "strings" +) + +const exampleToken = "EXAMPLE" + +const exampleLineStart = " +// +// ```yaml +// foo: +// bar: +// ``` +// +// [Download example](../../examples/guestbook/frontend-service.yaml?raw=true) +// +func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { + var err error + type exampleTag struct { + token string + linkText string + fileType string + } + exampleTags := []exampleTag{} + + // collect all example Tags + for _, mline := range mlines { + if mline.preformatted || !mline.beginTag { + continue + } + line := mline.data + if !strings.HasPrefix(line, exampleLineStart) { + continue + } + match := exampleMungeTagRE.FindStringSubmatch(line) + if len(match) < 4 { + err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line) + return mlines, err + } + tag := exampleTag{ + token: exampleToken + " " + match[1], + linkText: match[1], + fileType: match[3], + } + exampleTags = append(exampleTags, tag) + } + // update all example Tags + for _, tag := range exampleTags { + ft := "" + if tag.fileType == "json" { + ft = "json" + } + if tag.fileType == "yaml" { + ft = "yaml" + } + example, err := exampleContent(filePath, tag.linkText, ft) + if err != nil { + return mlines, err + } + mlines, err = updateMacroBlock(mlines, tag.token, example) + if err != nil { + return mlines, err + } + } + return mlines, nil +} + +// exampleContent retrieves the content of the file at linkPath +func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { + repoRel, err := makeRepoRelative(linkPath, filePath) + if err != nil { + return nil, err + } + + fileRel, err := makeFileRelative(linkPath, filePath) + if err != nil { + return nil, err + } + + dat, err := ioutil.ReadFile(repoRel) + if err != nil { + return nil, err + } + + // remove leading and trailing spaces and newlines + trimmedFileContent := strings.TrimSpace(string(dat)) + content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) + out := getMungeLines(content) + return out, nil +} diff --git a/_tools/mungedocs/example_syncer_test.go b/_tools/mungedocs/example_syncer_test.go new file mode 100644 index 0000000000000..3d51b28c9d819 --- /dev/null +++ b/_tools/mungedocs/example_syncer_test.go @@ -0,0 +1,68 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func Test_syncExamples(t *testing.T) { + var podExample = `apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 +` + var textExample = `some text +` + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", + }, + { + "\n\n", + "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", + }, + { + "\n\n", + "\n\n```\n" + textExample + "```\n\n[Download example](testdata/example.txt?raw=true)\n\n", + }, + } + repoRoot = "" + for _, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := syncExamples("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !expected.Equal(actual) { + t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String()) + } + } +} diff --git a/_tools/mungedocs/headers.go b/_tools/mungedocs/headers.go new file mode 100644 index 0000000000000..e23ae7536e5ee --- /dev/null +++ b/_tools/mungedocs/headers.go @@ -0,0 +1,74 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" +) + +var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`) + +func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines { + var out mungeLines + + mline := mlines[linenum] + line := mlines[linenum].data + + matches := headerRegex.FindStringSubmatch(line) + if matches == nil { + out = append(out, mline) + return out + } + + // There must be a blank line before the # (unless first line in file) + if linenum != 0 { + newlen := len(newlines) + if newlines[newlen-1].data != "" { + out = append(out, blankMungeLine) + } + } + + // There must be a space AFTER the ##'s + newline := fmt.Sprintf("%s %s", matches[1], matches[2]) + newmline := newMungeLine(newline) + out = append(out, newmline) + + // The next line needs to be a blank line (unless last line in file) + if len(mlines) > linenum+1 && mlines[linenum+1].data != "" { + out = append(out, blankMungeLine) + } + return out +} + +// Header lines need whitespace around them and after the #s. +func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + for i, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + if !mline.header { + out = append(out, mline) + continue + } + newLines := fixHeaderLine(mlines, out, i) + out = append(out, newLines...) + } + return out, nil +} diff --git a/_tools/mungedocs/headers_test.go b/_tools/mungedocs/headers_test.go new file mode 100644 index 0000000000000..d308b01034873 --- /dev/null +++ b/_tools/mungedocs/headers_test.go @@ -0,0 +1,74 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestHeaderLines(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "# ok", + "# ok", + }, + { + "## ok", + "## ok", + }, + { + "##### ok", + "##### ok", + }, + { + "##fix", + "## fix", + }, + { + "foo\n\n##fix\n\nbar", + "foo\n\n## fix\n\nbar", + }, + { + "foo\n##fix\nbar", + "foo\n\n## fix\n\nbar", + }, + { + "foo\n```\n##fix\n```\nbar", + "foo\n```\n##fix\n```\nbar", + }, + { + "foo\n#fix1\n##fix2\nbar", + "foo\n\n# fix1\n\n## fix2\n\nbar", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateHeaderLines("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} diff --git a/_tools/mungedocs/kubectl_dash_f.go b/_tools/mungedocs/kubectl_dash_f.go new file mode 100644 index 0000000000000..2dae48cc8574d --- /dev/null +++ b/_tools/mungedocs/kubectl_dash_f.go @@ -0,0 +1,125 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path" + "strings" +) + +// Looks for lines that have kubectl commands with -f flags and files that +// don't exist. +func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) { + var errors []string + for i, mline := range mlines { + if !mline.preformatted { + continue + } + if err := lookForKubectl(mline.data, i); err != nil { + errors = append(errors, err.Error()) + } + } + err := error(nil) + if len(errors) != 0 { + err = fmt.Errorf("%s", strings.Join(errors, "\n")) + } + return mlines, err +} + +func lookForKubectl(line string, lineNum int) error { + fields := strings.Fields(line) + for i := range fields { + if fields[i] == "kubectl" { + return gotKubectl(lineNum, fields, i) + } + } + return nil +} + +func gotKubectl(lineNum int, fields []string, fieldNum int) error { + for i := fieldNum + 1; i < len(fields); i++ { + switch fields[i] { + case "create", "update", "replace", "delete": + return gotCommand(lineNum, fields, i) + } + } + return nil +} + +func gotCommand(lineNum int, fields []string, fieldNum int) error { + for i := fieldNum + 1; i < len(fields); i++ { + if strings.HasPrefix(fields[i], "-f") { + return gotDashF(lineNum, fields, i) + } + } + return nil +} + +func gotDashF(lineNum int, fields []string, fieldNum int) error { + target := "" + if fields[fieldNum] == "-f" { + if fieldNum+1 == len(fields) { + return fmt.Errorf("ran out of fields after '-f'") + } + target = fields[fieldNum+1] + } else { + target = fields[fieldNum][2:] + } + // Turn dirs into file-like names. + target = strings.TrimRight(target, "/") + + // Now exclude special-cases + + if target == "-" || target == "FILENAME" { + // stdin and "FILENAME" are OK + return nil + } + if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { + // URLs are ok + return nil + } + if strings.HasPrefix(target, "./") { + // Same-dir files are usually created in the same example + return nil + } + if strings.HasPrefix(target, "~/") { + // Home directory may also be created by the same example + return nil + } + if strings.HasPrefix(target, "/") { + // Absolute paths tend to be /tmp/* and created in the same example. + return nil + } + if strings.HasPrefix(target, "$") { + // Allow the start of the target to be an environment + // variable that points to the root of the kubernetes + // repo. + split := strings.SplitN(target, "/", 2) + if len(split) == 2 { + target = split[1] + } + } + + // If we got here we expect the file to exist. + _, err := os.Stat(path.Join(repoRoot, target)) + if os.IsNotExist(err) { + return fmt.Errorf("%d: target file %q does not exist", lineNum, target) + } + return err +} diff --git a/_tools/mungedocs/kubectl_dash_f_test.go b/_tools/mungedocs/kubectl_dash_f_test.go new file mode 100644 index 0000000000000..6f18fd547a408 --- /dev/null +++ b/_tools/mungedocs/kubectl_dash_f_test.go @@ -0,0 +1,143 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import "testing" + +func TestKubectlDashF(t *testing.T) { + var cases = []struct { + in string + ok bool + }{ + // No match + {"", true}, + { + "Foo\nBar\n", + true, + }, + { + "Foo\nkubectl blah blech\nBar", + true, + }, + { + "Foo\n```shell\nkubectl blah blech\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create blech\n```\nBar", + true, + }, + // Special cases + { + "Foo\n```\nkubectl -blah create -f -\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f-\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f FILENAME\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fFILENAME\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f http://google.com/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fhttp://google.com/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f ./foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f./foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f /foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f/foobar\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -f~/foobar\n```\nBar", + true, + }, + // Real checks + { + "Foo\n```\nkubectl -blah create -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah create -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah update -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah update -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah replace -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah replace -fmungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah delete -f mungedocs.go\n```\nBar", + true, + }, + { + "Foo\n```\nkubectl -blah delete -fmungedocs.go\n```\nBar", + true, + }, + // Failures + { + "Foo\n```\nkubectl -blah delete -f does_not_exist\n```\nBar", + false, + }, + { + "Foo\n```\nkubectl -blah delete -fdoes_not_exist\n```\nBar", + false, + }, + } + for i, c := range cases { + repoRoot = "" + in := getMungeLines(c.in) + _, err := updateKubectlFileTargets("filename.md", in) + if err != nil && c.ok { + t.Errorf("case[%d]: expected success, got %v", i, err) + } + if err == nil && !c.ok { + t.Errorf("case[%d]: unexpected success", i) + } + } +} diff --git a/_tools/mungedocs/links.go b/_tools/mungedocs/links.go new file mode 100644 index 0000000000000..c05cdfa1874b7 --- /dev/null +++ b/_tools/mungedocs/links.go @@ -0,0 +1,238 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "fmt" + "net/url" + "os" + "path" + "regexp" + "strings" +) + +var ( + // Finds markdown links of the form [foo](bar "alt-text"). + linkRE = regexp.MustCompile(`\[([^]]*)\]\(([^)]*)\)`) + // Finds markdown link typos of the form (foo)[bar] + badLinkRE = regexp.MustCompile(`\([^]()]*\)\[[^]()]*\]`) + // Splits the link target into link target and alt-text. + altTextRE = regexp.MustCompile(`([^)]*)( ".*")`) +) + +func processLink(in string, filePath string) (string, error) { + var errs []string + out := linkRE.ReplaceAllStringFunc(in, func(in string) string { + var err error + match := linkRE.FindStringSubmatch(in) + if match == nil { + errs = append(errs, fmt.Sprintf("Detected this line had a link, but unable to parse, %v", in)) + return "" + } + // match[0] is the entire expression; + visibleText := match[1] + linkText := match[2] + altText := "" + if parts := altTextRE.FindStringSubmatch(linkText); parts != nil { + linkText = parts[1] + altText = parts[2] + } + + // clean up some random garbage I found in our docs. + linkText = strings.Trim(linkText, " ") + linkText = strings.Trim(linkText, "\n") + linkText = strings.Trim(linkText, " ") + + u, terr := url.Parse(linkText) + if terr != nil { + errs = append(errs, fmt.Sprintf("link %q is unparsable: %v", linkText, terr)) + return in + } + + if u.Host != "" && u.Host != "github.com" { + // We only care about relative links and links within github. + return in + } + + suggestedVisibleText := visibleText + if u.Path != "" && !strings.HasPrefix(linkText, "TODO:") { + newPath, targetExists := checkPath(filePath, path.Clean(u.Path)) + if !targetExists { + errs = append(errs, fmt.Sprintf("%q: target not found", linkText)) + return in + } + u.Path = newPath + if strings.HasPrefix(u.Path, "/") { + u.Host = "github.com" + u.Scheme = "https" + } else { + // Remove host and scheme from relative paths + u.Host = "" + u.Scheme = "" + } + // Make the visible text show the absolute path if it's + // not nested in or beneath the current directory. + if strings.HasPrefix(u.Path, "..") { + dir := path.Dir(filePath) + suggestedVisibleText, err = makeRepoRelative(path.Join(dir, u.Path), filePath) + if err != nil { + errs = append(errs, fmt.Sprintf("%q: unable to make path relative", filePath)) + return in + } + } else { + suggestedVisibleText = u.Path + } + var unescaped string + if unescaped, err = url.QueryUnescape(u.String()); err != nil { + // Remove %28 type stuff, be nice to humans. + // And don't fight with the toc generator. + linkText = unescaped + } else { + linkText = u.String() + } + } + // If the current visible text is trying to be a file name, use + // the correct file name. + if strings.HasSuffix(visibleText, ".md") && !strings.ContainsAny(visibleText, ` '"`+"`") { + visibleText = suggestedVisibleText + } + + return fmt.Sprintf("[%s](%s)", visibleText, linkText+altText) + }) + if len(errs) != 0 { + return "", errors.New(strings.Join(errs, ",")) + } + return out, nil +} + +// updateLinks assumes lines has links in markdown syntax, and verifies that +// any relative links actually point to files that exist. +func updateLinks(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + allErrs := []string{} + + for lineNum, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + if badMatch := badLinkRE.FindString(mline.data); badMatch != "" { + allErrs = append(allErrs, + fmt.Sprintf("On line %d: found backwards markdown link %q", lineNum, badMatch)) + } + if !mline.link { + out = append(out, mline) + continue + } + line, err := processLink(mline.data, filePath) + if err != nil { + var s = fmt.Sprintf("On line %d: %s", lineNum, err.Error()) + err := errors.New(s) + allErrs = append(allErrs, err.Error()) + } + ml := newMungeLine(line) + out = append(out, ml) + } + err := error(nil) + if len(allErrs) != 0 { + err = fmt.Errorf("%s", strings.Join(allErrs, "\n")) + } + return out, err +} + +// We have to append together before path.Clean will be able to tell that stuff +// like ../docs isn't needed. +func cleanPath(dirPath, linkPath string) string { + clean := path.Clean(path.Join(dirPath, linkPath)) + if strings.HasPrefix(clean, dirPath+"/") { + out := strings.TrimPrefix(clean, dirPath+"/") + if out != linkPath { + fmt.Printf("%s -> %s\n", linkPath, out) + } + return out + } + return linkPath +} + +func checkPath(filePath, linkPath string) (newPath string, ok bool) { + dir := path.Dir(filePath) + absFilePrefixes := []string{ + "/kubernetes/kubernetes/blob/master/", + "/kubernetes/kubernetes/tree/master/", + } + for _, prefix := range absFilePrefixes { + if strings.HasPrefix(linkPath, prefix) { + linkPath = strings.TrimPrefix(linkPath, prefix) + // Now linkPath is relative to the root of the repo. The below + // loop that adds ../ at the beginning of the path should find + // the right path. + break + } + } + if strings.HasPrefix(linkPath, "/") { + // These links might go to e.g. the github issues page, or a + // file at a particular revision, or another github project + // entirely. + return linkPath, true + } + linkPath = cleanPath(dir, linkPath) + + // Fast exit if the link is already correct. + if info, err := os.Stat(path.Join(dir, linkPath)); err == nil { + if info.IsDir() { + return linkPath + "/", true + } + return linkPath, true + } + + for strings.HasPrefix(linkPath, "../") { + linkPath = strings.TrimPrefix(linkPath, "../") + } + + // Fix - vs _ automatically + nameMungers := []func(string) string{ + func(s string) string { return s }, + func(s string) string { return strings.Replace(s, "-", "_", -1) }, + func(s string) string { return strings.Replace(s, "_", "-", -1) }, + } + // Fix being moved into/out of admin (replace "admin" with directory + // you're doing mass movements to/from). + pathMungers := []func(string) string{ + func(s string) string { return s }, + func(s string) string { return path.Join("admin", s) }, + func(s string) string { return strings.TrimPrefix(s, "admin/") }, + } + + for _, namer := range nameMungers { + for _, pather := range pathMungers { + newPath = pather(namer(linkPath)) + for i := 0; i < 7; i++ { + // The file must exist. + target := path.Join(dir, newPath) + if info, err := os.Stat(target); err == nil { + if info.IsDir() { + return newPath + "/", true + } + return cleanPath(dir, newPath), true + } + newPath = path.Join("..", newPath) + } + } + } + return linkPath, false +} diff --git a/_tools/mungedocs/links_test.go b/_tools/mungedocs/links_test.go new file mode 100644 index 0000000000000..37f8e53171423 --- /dev/null +++ b/_tools/mungedocs/links_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "testing" +) + +var _ = fmt.Printf + +func TestBadLinks(t *testing.T) { + var cases = []struct { + in string + }{ + {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/NOTREADME.md)"}, + {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/docs/NOTREADME.md)"}, + {"[NOTREADME](../NOTREADME.md)"}, + } + for _, c := range cases { + in := getMungeLines(c.in) + _, err := updateLinks("filename.md", in) + if err == nil { + t.Errorf("Expected error") + } + } +} +func TestGoodLinks(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + {"[README](https://lwn.net)", + "[README](https://lwn.net)"}, + // _ to - + {"[README](https://github.com/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/test_dashes.md)", + "[README](../../cmd/mungedocs/testdata/test-dashes.md)"}, + // - to _ + {"[README](../../cmd/mungedocs/testdata/test-underscores.md)", + "[README](../../cmd/mungedocs/testdata/test_underscores.md)"}, + + // Does this even make sense? i dunno + {"[README](/docs/README.md)", + "[README](https://github.com/docs/README.md)"}, + {"[README](/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/README.md)", + "[README](../../cmd/mungedocs/testdata/README.md)"}, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateLinks("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} diff --git a/_tools/mungedocs/mungedocs.go b/_tools/mungedocs/mungedocs.go new file mode 100644 index 0000000000000..ef3eb2da1a5e6 --- /dev/null +++ b/_tools/mungedocs/mungedocs.go @@ -0,0 +1,235 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "os" + "path" + "path/filepath" + "strings" +) + +var ( + verbose = flag.Bool("verbose", false, "On verification failure, emit pre-munge and post-munge versions.") + verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") + norecurse = flag.Bool("norecurse", false, "Only process the files of --root-dir.") + upstream = flag.String("upstream", "upstream", "The name of the upstream Git remote to pull from") + rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") + // "repo-root" seems like a dumb name, this is the relative path (from rootDir) to get to the repoRoot + relRoot = flag.String("repo-root", "..", `Appended to --root-dir to get the repository root. +It's done this way so that generally you just have to set --root-dir. +Examples: + * --root-dir=docs/ --repo-root=.. means the repository root is ./ + * --root-dir=/usr/local/long/path/repo/docs/ --repo-root=.. means the repository root is /usr/local/long/path/repo/ + * --root-dir=/usr/local/long/path/repo/docs/admin --repo-root=../.. means the repository root is /usr/local/long/path/repo/`) + skipMunges = flag.String("skip-munges", "", "Comma-separated list of munges to *not* run. Available munges are: "+availableMungeList) + repoRoot string + + ErrChangesNeeded = errors.New("mungedocs: changes required") + + // All of the munge operations to perform. + // TODO: allow selection from command line. (e.g., just check links in the examples directory.) + allMunges = []munge{ + // Simple "check something" functions must run first. + {"preformat-balance", checkPreformatBalance}, + // Functions which modify state. + {"remove-whitespace", updateWhitespace}, + {"table-of-contents", updateTOC}, + {"md-links", updateLinks}, + {"blank-lines-surround-preformatted", updatePreformatted}, + {"header-lines", updateHeaderLines}, + {"analytics", updateAnalytics}, + {"kubectl-dash-f", updateKubectlFileTargets}, + {"sync-examples", syncExamples}, + } + availableMungeList = func() string { + names := []string{} + for _, m := range allMunges { + names = append(names, m.name) + } + return strings.Join(names, ",") + }() +) + +// a munge processes a document, returning an updated document xor an error. +// The fn is NOT allowed to mutate 'before', if changes are needed it must copy +// data into a new byte array and return that. +type munge struct { + name string + fn func(filePath string, mlines mungeLines) (after mungeLines, err error) +} + +type fileProcessor struct { + // Which munge functions should we call? + munges []munge + + // Are we allowed to make changes? + verifyOnly bool +} + +// Either change a file or verify that it needs no changes (according to modify argument) +func (f fileProcessor) visit(path string) error { + if !strings.HasSuffix(path, ".md") { + return nil + } + + fileBytes, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + mungeLines := getMungeLines(string(fileBytes)) + + modificationsMade := false + errFound := false + filePrinted := false + for _, munge := range f.munges { + after, err := munge.fn(path, mungeLines) + if err != nil || !after.Equal(mungeLines) { + if !filePrinted { + fmt.Printf("%s\n----\n", path) + filePrinted = true + } + fmt.Printf("%s:\n", munge.name) + if *verbose { + if len(mungeLines) <= 20 { + fmt.Printf("INPUT: <<<%v>>>\n", mungeLines) + fmt.Printf("MUNGED: <<<%v>>>\n", after) + } else { + fmt.Printf("not printing failed chunk: too many lines\n") + } + } + if err != nil { + fmt.Println(err) + errFound = true + } else { + fmt.Println("contents were modified") + modificationsMade = true + } + fmt.Println("") + } + mungeLines = after + } + + // Write out new file with any changes. + if modificationsMade { + if f.verifyOnly { + // We're not allowed to make changes. + return ErrChangesNeeded + } + ioutil.WriteFile(path, mungeLines.Bytes(), 0644) + } + if errFound { + return ErrChangesNeeded + } + + return nil +} + +func newWalkFunc(fp *fileProcessor, changesNeeded *bool) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + stat, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if path != *rootDir && stat.IsDir() && *norecurse { + return filepath.SkipDir + } + if err := fp.visit(path); err != nil { + *changesNeeded = true + if err != ErrChangesNeeded { + return err + } + } + return nil + } +} + +func wantedMunges() (filtered []munge) { + skipList := strings.Split(*skipMunges, ",") + skipped := map[string]bool{} + for _, m := range skipList { + if len(m) > 0 { + skipped[m] = true + } + } + for _, m := range allMunges { + if !skipped[m.name] { + filtered = append(filtered, m) + } else { + // Remove from the map so we can verify that everything + // requested was in fact valid. + delete(skipped, m.name) + } + } + if len(skipped) != 0 { + fmt.Fprintf(os.Stderr, "ERROR: requested to skip %v, but these are not valid munges. (valid: %v)\n", skipped, availableMungeList) + os.Exit(1) + } + return filtered +} + +func main() { + var err error + flag.Parse() + + if *rootDir == "" { + fmt.Fprintf(os.Stderr, "usage: %s [--help] [--verify] [--norecurse] --root-dir [--skip-munges=] [--upstream=] \n", flag.Arg(0)) + os.Exit(1) + } + + repoRoot = path.Join(*rootDir, *relRoot) + repoRoot, err = filepath.Abs(repoRoot) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(2) + } + + fp := fileProcessor{ + munges: wantedMunges(), + verifyOnly: *verify, + } + + // For each markdown file under source docs root, process the doc. + // - If any error occurs: exit with failure (exit >1). + // - If verify is true: exit 0 if no changes needed, exit 1 if changes + // needed. + // - If verify is false: exit 0 if changes successfully made or no + // changes needed, exit 1 if manual changes are needed. + var changesNeeded bool + + err = filepath.Walk(*rootDir, newWalkFunc(&fp, &changesNeeded)) + if err != nil { + fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) + os.Exit(2) + } + if changesNeeded { + if *verify { + fmt.Fprintf(os.Stderr, "FAIL: changes needed but not made due to --verify\n") + } else { + fmt.Fprintf(os.Stderr, "FAIL: some manual changes are still required.\n") + } + os.Exit(1) + } +} diff --git a/_tools/mungedocs/preformatted.go b/_tools/mungedocs/preformatted.go new file mode 100644 index 0000000000000..582ba981a109d --- /dev/null +++ b/_tools/mungedocs/preformatted.go @@ -0,0 +1,51 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import "fmt" + +// Blocks of ``` need to have blank lines on both sides or they don't look +// right in HTML. +func updatePreformatted(filePath string, mlines mungeLines) (mungeLines, error) { + var out mungeLines + inpreformat := false + for i, mline := range mlines { + if !inpreformat && mline.preformatted { + if i == 0 || out[len(out)-1].data != "" { + out = append(out, blankMungeLine) + } + // start of a preformat block + inpreformat = true + } + out = append(out, mline) + if inpreformat && !mline.preformatted { + if i >= len(mlines)-2 || mlines[i+1].data != "" { + out = append(out, blankMungeLine) + } + inpreformat = false + } + } + return out, nil +} + +// If the file ends on a preformatted line, there must have been an imbalance. +func checkPreformatBalance(filePath string, mlines mungeLines) (mungeLines, error) { + if len(mlines) > 0 && mlines[len(mlines)-1].preformatted { + return mlines, fmt.Errorf("unbalanced triple backtick delimiters") + } + return mlines, nil +} diff --git a/_tools/mungedocs/preformatted_test.go b/_tools/mungedocs/preformatted_test.go new file mode 100644 index 0000000000000..ec93854ba072f --- /dev/null +++ b/_tools/mungedocs/preformatted_test.go @@ -0,0 +1,98 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func TestPreformatted(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "```\nbob\n```", + "\n```\nbob\n```\n\n", + }, + { + "```\nbob\n```\n```\nnotbob\n```\n", + "\n```\nbob\n```\n\n```\nnotbob\n```\n\n", + }, + { + "```bob```\n", + "```bob```\n", + }, + { + " ```\n bob\n ```", + "\n ```\n bob\n ```\n\n", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updatePreformatted("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + + if !actual.Equal(expected) { + t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) + } + } +} + +func TestPreformattedImbalance(t *testing.T) { + var cases = []struct { + in string + ok bool + }{ + {"", true}, + {"```\nin\n```", true}, + {"```\nin\n```\nout", true}, + {"```", false}, + {"```\nin\n```\nout\n```", false}, + } + for i, c := range cases { + in := getMungeLines(c.in) + out, err := checkPreformatBalance("filename.md", in) + if err != nil && c.ok { + t.Errorf("case[%d]: expected success", i) + } + if err == nil && !c.ok { + t.Errorf("case[%d]: expected failure", i) + } + // Even in case of misformat, return all the text, + // so that the user's work is not lost. + if !equalMungeLines(out, in) { + t.Errorf("case[%d]: expected munged text to be identical to input text", i) + } + } +} + +func equalMungeLines(a, b mungeLines) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/_tools/mungedocs/testdata/README.md b/_tools/mungedocs/testdata/README.md new file mode 100644 index 0000000000000..7b57bd29ea8af --- /dev/null +++ b/_tools/mungedocs/testdata/README.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/example.txt b/_tools/mungedocs/testdata/example.txt new file mode 100644 index 0000000000000..7b57bd29ea8af --- /dev/null +++ b/_tools/mungedocs/testdata/example.txt @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/pod.yaml b/_tools/mungedocs/testdata/pod.yaml new file mode 100644 index 0000000000000..89920b83a9af6 --- /dev/null +++ b/_tools/mungedocs/testdata/pod.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nginx +spec: + containers: + - name: nginx + image: nginx + ports: + - containerPort: 80 \ No newline at end of file diff --git a/_tools/mungedocs/testdata/test-dashes.md b/_tools/mungedocs/testdata/test-dashes.md new file mode 100644 index 0000000000000..7b57bd29ea8af --- /dev/null +++ b/_tools/mungedocs/testdata/test-dashes.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/testdata/test_underscores.md b/_tools/mungedocs/testdata/test_underscores.md new file mode 100644 index 0000000000000..7b57bd29ea8af --- /dev/null +++ b/_tools/mungedocs/testdata/test_underscores.md @@ -0,0 +1 @@ +some text diff --git a/_tools/mungedocs/toc.go b/_tools/mungedocs/toc.go new file mode 100644 index 0000000000000..649a954bf5abc --- /dev/null +++ b/_tools/mungedocs/toc.go @@ -0,0 +1,89 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "regexp" + "strings" +) + +const tocMungeTag = "GENERATED_TOC" + +var r = regexp.MustCompile("[^A-Za-z0-9-]") + +// inserts/updates a table of contents in markdown file. +// +// First, builds a ToC. +// Then, finds the magic macro block tags and replaces anything between those with +// the ToC, thereby updating any previously inserted ToC. +// +// TODO(erictune): put this in own package with tests +func updateTOC(filePath string, mlines mungeLines) (mungeLines, error) { + toc := buildTOC(mlines) + updatedMarkdown, err := updateMacroBlock(mlines, tocMungeTag, toc) + if err != nil { + return mlines, err + } + return updatedMarkdown, nil +} + +// builds table of contents for markdown file +// +// First scans for all section headers (lines that begin with "#" but not within code quotes) +// and builds a table of contents from those. Assumes bookmarks for those will be +// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. +// builds the ToC. + +func buildTOC(mlines mungeLines) mungeLines { + var out mungeLines + bookmarks := map[string]int{} + + for _, mline := range mlines { + if mline.preformatted || !mline.header { + continue + } + // Add a blank line after the munge start tag + if len(out) == 0 { + out = append(out, blankMungeLine) + } + line := mline.data + noSharps := strings.TrimLeft(line, "#") + numSharps := len(line) - len(noSharps) + heading := strings.Trim(noSharps, " \n") + if numSharps > 0 { + indent := strings.Repeat(" ", numSharps-1) + bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) + // remove symbols (except for -) in bookmarks + bookmark = r.ReplaceAllString(bookmark, "") + // Incremental counter for duplicate bookmarks + next := bookmarks[bookmark] + bookmarks[bookmark] = next + 1 + if next > 0 { + bookmark = fmt.Sprintf("%s-%d", bookmark, next) + } + tocLine := fmt.Sprintf("%s- [%s](#%s)", indent, heading, bookmark) + out = append(out, newMungeLine(tocLine)) + } + + } + // Add a blank line before the munge end tag + if len(out) != 0 { + out = append(out, blankMungeLine) + } + return out +} diff --git a/_tools/mungedocs/toc_test.go b/_tools/mungedocs/toc_test.go new file mode 100644 index 0000000000000..95ff354ed5646 --- /dev/null +++ b/_tools/mungedocs/toc_test.go @@ -0,0 +1,79 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "testing" +) + +func Test_buildTOC(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + {"Lorem ipsum\ndolor sit amet\n", ""}, + { + "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", + }, + { + "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n```bash\n#!/bin/sh\n```", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", + }, + { + "# Title\nLorem ipsum \n## Section Heading\n### Ok, why doesn't this work? ...add 4 *more* `symbols`!\ndolor sit amet\n", + "\n- [Title](#title)\n - [Section Heading](#section-heading)\n - [Ok, why doesn't this work? ...add 4 *more* `symbols`!](#ok-why-doesnt-this-work-add-4-more-symbols)\n\n", + }, + } + for i, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual := buildTOC(in) + if !expected.Equal(actual) { + t.Errorf("Case[%d] Expected TOC '%v' but got '%v'", i, expected.String(), actual.String()) + } + } +} + +func Test_updateTOC(t *testing.T) { + var cases = []struct { + in string + expected string + }{ + {"", ""}, + { + "Lorem ipsum\ndolor sit amet\n", + "Lorem ipsum\ndolor sit amet\n", + }, + { + "# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", + "# Title\nLorem ipsum \n**table of contents**\n\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n", + }, + } + for _, c := range cases { + in := getMungeLines(c.in) + expected := getMungeLines(c.expected) + actual, err := updateTOC("filename.md", in) + if err != nil { + t.Errorf("Error: %v", err) + } + if !expected.Equal(actual) { + t.Errorf("Expected TOC '%v' but got '%v'", expected.String(), actual.String()) + } + } +} diff --git a/_tools/mungedocs/util.go b/_tools/mungedocs/util.go new file mode 100644 index 0000000000000..c25e1d1976ecc --- /dev/null +++ b/_tools/mungedocs/util.go @@ -0,0 +1,291 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "fmt" + "path" + "path/filepath" + "regexp" + "strings" + "unicode" +) + +// Replaces the text between matching "beginMark" and "endMark" within the +// document represented by "lines" with "insertThis". +// +// Delimiters should occupy own line. +// Returns copy of document with modifications. +func updateMacroBlock(mlines mungeLines, token string, insertThis mungeLines) (mungeLines, error) { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + var out mungeLines + betweenBeginAndEnd := false + for _, mline := range mlines { + if mline.preformatted && !betweenBeginAndEnd { + out = append(out, mline) + continue + } + line := mline.data + if mline.beginTag && line == beginMark { + if betweenBeginAndEnd { + return nil, fmt.Errorf("found second begin mark while updating macro blocks") + } + betweenBeginAndEnd = true + out = append(out, mline) + } else if mline.endTag && line == endMark { + if !betweenBeginAndEnd { + return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") + } + betweenBeginAndEnd = false + out = append(out, insertThis...) + out = append(out, mline) + } else { + if !betweenBeginAndEnd { + out = append(out, mline) + } + } + } + if betweenBeginAndEnd { + return nil, fmt.Errorf("never found closing end mark while updating macro blocks") + } + return out, nil +} + +// Tests that a document, represented as a slice of lines, has a line. Ignores +// leading and trailing space. +func hasLine(lines mungeLines, needle string) bool { + for _, mline := range lines { + haystack := strings.TrimSpace(mline.data) + if haystack == needle { + return true + } + } + return false +} + +func removeMacroBlock(token string, mlines mungeLines) (mungeLines, error) { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + var out mungeLines + betweenBeginAndEnd := false + for _, mline := range mlines { + if mline.preformatted { + out = append(out, mline) + continue + } + line := mline.data + if mline.beginTag && line == beginMark { + if betweenBeginAndEnd { + return nil, fmt.Errorf("found second begin mark while updating macro blocks") + } + betweenBeginAndEnd = true + } else if mline.endTag && line == endMark { + if !betweenBeginAndEnd { + return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") + } + betweenBeginAndEnd = false + } else { + if !betweenBeginAndEnd { + out = append(out, mline) + } + } + } + if betweenBeginAndEnd { + return nil, fmt.Errorf("never found closing end mark while updating macro blocks") + } + return out, nil +} + +// Add a macro block to the beginning of a set of lines +func prependMacroBlock(token string, mlines mungeLines) mungeLines { + beginLine := newMungeLine(beginMungeTag(token)) + endLine := newMungeLine(endMungeTag(token)) + out := mungeLines{beginLine, endLine} + if len(mlines) > 0 && mlines[0].data != "" { + out = append(out, blankMungeLine) + } + return append(out, mlines...) +} + +// Add a macro block to the end of a set of lines +func appendMacroBlock(mlines mungeLines, token string) mungeLines { + beginLine := newMungeLine(beginMungeTag(token)) + endLine := newMungeLine(endMungeTag(token)) + out := mlines + if len(mlines) > 0 && mlines[len(mlines)-1].data != "" { + out = append(out, blankMungeLine) + } + return append(out, beginLine, endLine) +} + +// Tests that a document, represented as a slice of lines, has a macro block. +func hasMacroBlock(lines mungeLines, token string) bool { + beginMark := beginMungeTag(token) + endMark := endMungeTag(token) + + foundBegin := false + for _, mline := range lines { + if mline.preformatted { + continue + } + if !mline.beginTag && !mline.endTag { + continue + } + line := mline.data + switch { + case !foundBegin && line == beginMark: + foundBegin = true + case foundBegin && line == endMark: + return true + } + } + return false +} + +// Returns the canonical begin-tag for a given description. This does not +// include the trailing newline. +func beginMungeTag(desc string) string { + return fmt.Sprintf("", desc) +} + +// Returns the canonical end-tag for a given description. This does not +// include the trailing newline. +func endMungeTag(desc string) string { + return fmt.Sprintf("", desc) +} + +type mungeLine struct { + data string + preformatted bool + header bool + link bool + beginTag bool + endTag bool +} + +type mungeLines []mungeLine + +func (m1 mungeLines) Equal(m2 mungeLines) bool { + if len(m1) != len(m2) { + return false + } + for i := range m1 { + if m1[i].data != m2[i].data { + return false + } + } + return true +} + +func (mlines mungeLines) String() string { + slice := []string{} + for _, mline := range mlines { + slice = append(slice, mline.data) + } + s := strings.Join(slice, "\n") + // We need to tack on an extra newline at the end of the file + return s + "\n" +} + +func (mlines mungeLines) Bytes() []byte { + return []byte(mlines.String()) +} + +var ( + // Finds all preformatted block start/stops. + preformatRE = regexp.MustCompile("^\\s*```") + notPreformatRE = regexp.MustCompile("^\\s*```.*```") + // Is this line a header? + mlHeaderRE = regexp.MustCompile(`^#`) + // Is there a link on this line? + mlLinkRE = regexp.MustCompile(`\[[^]]*\]\([^)]*\)`) + beginTagRE = regexp.MustCompile(` + +{% assign glossary_terms = site.data.glossary | where_exp: "term", "term.id != '_example'" %} + +{% assign tag_map = "" | split: " " %} + +{% for tag in site.data.canonical-tags %} + +{% assign term_list = glossary_terms | where_exp:"term", "term.tags contains tag" | map: "id" %} + +{% assign tag_obj = "" | split: " " | push: tag | push: term_list %} + +{% assign tag_map = tag_map | push: tag_obj %} + +{% endfor %} diff --git a/_includes/templates/glossary/README.md b/_includes/templates/glossary/README.md new file mode 100644 index 0000000000000..9d3d49d48eaf7 --- /dev/null +++ b/_includes/templates/glossary/README.md @@ -0,0 +1,62 @@ +# Kubernetes Glossary + +To write a glossary snippet, start with a copy of the template, [`/_data/glossary/_example.yml`](/_data/glossary/_example.yml). Make sure to provide (or omit) values for the following fields: + +* (Required) `id`. + * This field must match the name of the glossary file itself (without the `*.yml` extension). It is *not* intended to be displayed to users, and is only used programmatically. +* (Required) `name`. + * The name of the term. +* (Required) `tags`. + * Must be one of the tags listed in kubernetes.github.io/_data/canonical-terms-tags.yml. +* (Required) `short description`. + * Make sure to replace the instructional text in the template with your content. +* (Optional) `formerly` and `related`. + * If you do not provide these values, remove the fields. +* (Optional) `long description`. + * If you do not provide a long description, remove the field -- that is, the complete key-value pair. + +The `_example.yml` template also contains basic information about how to write your snippet. For additional guidance, continue reading this readme. + +## Glossary snippet style guide + +This style guide supplements the guidance provided in the glossary template. It's intended to help you think about what and how to write glossary definitions. For more general guidance on style, consult [the core docs style guide](https://kubernetes.io/docs/home/contribute/style-guide/). + +### Minimum viable snippet: + +Every snippet must include at least the short description. The long description is optional, but should be provided for terms that need additional clarification. For consistency with existing *Concept* definitions, *write your definitions as if the term is plural*. + +**short-description** (Required): One line (or two short lines) that provides a minimum definition. Do not repeat the term. Prefer fragments. Model after tooltips. End with a period. + +**long-description** (Optional): Longer additional text to appear after (in conjunction with) short description. Provide in cases where the short description is not sufficient for the intro paragraph to a topic. Write complete but concise sentences. + +### Examples + +```yaml +- name: Pod +- tags: + - Fundamental + - Workload + - API Object +- short-description: The smallest and simplest Kubernetes objects. Represent a set of running processes on your cluster. +- long-description: Pods most often run only a single container, and are managed by a Deployment. +``` + +```yaml +- name: Deployment +- tags: + - Fundamental + - Workload + - API Object +- short-description: Controllers that provide declarative updates for Pods and ReplicaSets. +- long-description: Deployments are responsible for creating and updating instances of an application. +``` + +### Thinking about definitions + +* **Think of the short description as it would appear in a tooltip.** Is it sufficient to get the reader started? Is it short enough to be read inside a small UI element? + + *Tip*: look at the API reference doc content (for example, https://kubernetes.io/docs/api-reference/v1.7/). Note, however, that this content should be used with care. The concept docs for Pod, for example, are clearer than the reference docs. + +* **The long description should follow the short description to make a complete introduction to a topic.** (This is the content that appears at the top of the content, before any generated TOC.) Does it provide information that's not already clear from the short description? Does it provide information that readers should have a general sense of before they dive into the details of the topic it helps introduce? + + *Tip:* the long description does not need to be long; it's intended to extend but not replace the short description. Look through current related docs for ideas. (The Deployment long description is taken from a tutorial, for example.) diff --git a/_includes/templates/glossary/_error.md b/_includes/templates/glossary/_error.md new file mode 100644 index 0000000000000..75ede4d120882 --- /dev/null +++ b/_includes/templates/glossary/_error.md @@ -0,0 +1,10 @@ +### ERROR: You must define a `{{ include.missing_block }}` field +{: style="color:red" } + +The glossary template requires that you provide text that {{ include.purpose }}. + +To get rid of this message and take advantage of this template, define the `{{ include.missing_block }}` +field for the `{{ include.term }}` glossary term and populate it with content. + +See `_data/glossary/_example.yml` for reference. + diff --git a/_includes/templates/glossary/snippet.md b/_includes/templates/glossary/snippet.md new file mode 100644 index 0000000000000..4ceea6c54930e --- /dev/null +++ b/_includes/templates/glossary/snippet.md @@ -0,0 +1,25 @@ +{% assign term_data = site.data.glossary.[include.term] %} + +{% if term_data.short-description %} + +{{ term_data.short-description | markdownify }} + +{% else %} + +{% include templates/glossary/_error.md term=term_data.name missing_block='short-description' purpose='concisely describes the key term in 1-2 lines' %} + +{% endif %} + +{% if include.length == "long" %} + +{% if term_data.long-description %} + +{{ term_data.long-description | markdownify }} + +{% else %} + +{% include templates/glossary/_error.md term=term_data.name missing_block='long-description' purpose='describes the key term in greater depth, supplementing the short-description' %} + +{% endif %} + +{% endif %} diff --git a/docs/concepts/workloads/controllers/statefulset.md b/docs/concepts/workloads/controllers/statefulset.md index 7f017a0bef280..d52a8c1e1f2e3 100644 --- a/docs/concepts/workloads/controllers/statefulset.md +++ b/docs/concepts/workloads/controllers/statefulset.md @@ -15,8 +15,7 @@ PetSets feature from 1.4. Users of PetSets are referred to the 1.5 [Upgrade Guide](/docs/tasks/manage-stateful-set/upgrade-pet-set-to-stateful-set/) for further information on how to upgrade existing PetSets to StatefulSets.** -A StatefulSet is a Controller that provides a unique identity to its Pods. It provides -guarantees about the ordering of deployment and scaling. +{% include templates/glossary/snippet.md term="statefulset" length="long" %} {% endcapture %} {% capture body %} @@ -40,6 +39,7 @@ provides a set of stateless replicas. Controllers such as [ReplicaSet](/docs/concepts/workloads/controllers/replicaset/) may be better suited to your stateless needs. ## Limitations + * StatefulSet is a beta resource, not available in any Kubernetes release prior to 1.5. * As with all alpha/beta resources, you can disable StatefulSet through the `--runtime-config` option passed to the apiserver. * The storage for a given Pod must either be provisioned by a [PersistentVolume Provisioner](http://releases.k8s.io/{{page.githubbranch}}/examples/persistent-volume-provisioning/README.md) based on the requested `storage class`, or pre-provisioned by an admin. diff --git a/test/glossary_test.go b/test/glossary_test.go new file mode 100644 index 0000000000000..df51025278ed6 --- /dev/null +++ b/test/glossary_test.go @@ -0,0 +1,91 @@ +/* +Copyright 2016 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package examples_test + +import ( + "io/ioutil" + "gopkg.in/yaml.v2" + "path" + "strings" + "testing" +) + +// Not unmarshaling short-description and long-description fields +// (for simplicity) +type GlossaryTerm struct { + Id string `yaml: "id"` + Name string `yaml: "name"` + Formerly []string `yaml: "formerly"` + Related []string `yaml: "related"` + Tags []string `yaml: "tags"` +} + +// Checks that all glossary files (../_data/glossary/*) contain valid tags +// that are present in the canonical set. +func TestCanonicalTags(t *testing.T) { + canonicalTagsFile := "../_data/canonical-tags.yml" + data, err := ioutil.ReadFile(canonicalTagsFile) + if err != nil { + t.Errorf("Unable to read file %s: %v", canonicalTagsFile, err) + return + } + var tagList []string + err = yaml.Unmarshal(data, &tagList) + if err != nil { + t.Errorf("Unable to unmarshal file %s: %v", tagList, err) + return + } + + canonicalTagsSet := make(map[string]bool) + for _, tag := range tagList { + canonicalTagsSet[tag] = true + } + + glossaryDir := "../_data/glossary" + files, err := ioutil.ReadDir(glossaryDir) + if err != nil { + t.Errorf("Unable to read directory %s: %v", glossaryDir, err) + return + } + + var term GlossaryTerm + for _, f := range files { + // skip validation of example files + if (strings.HasPrefix(f.Name(), "_")) { + continue + } + + filePath := path.Join(glossaryDir, f.Name()) + data, err := ioutil.ReadFile(filePath) + if err != nil { + t.Errorf("Unable to read file %s: %v", filePath, err) + continue + } + err = yaml.Unmarshal(data, &term) + if err != nil { + t.Errorf("Unable to unmarshal file %s: %v", filePath, err) + continue + } + + for _, tag := range term.Tags { + if _, present := canonicalTagsSet[tag]; !present { + t.Errorf("Glossary term \"%s\" has invalid tag \"%s\". See %s for the list of valid tags.", term.Name, tag, canonicalTagsFile) + continue + } + } + } +} From 082b5d0403bd3cba808c56dc37e25eed363f7b83 Mon Sep 17 00:00:00 2001 From: chenhuan12 Date: Thu, 24 Aug 2017 19:52:20 +0800 Subject: [PATCH 17/27] fix typo fix typo --- .../declarative-object-management-configuration.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/tutorials/object-management-kubectl/declarative-object-management-configuration.md b/docs/tutorials/object-management-kubectl/declarative-object-management-configuration.md index 74189782fd4ac..4e3f4788c97df 100644 --- a/docs/tutorials/object-management-kubectl/declarative-object-management-configuration.md +++ b/docs/tutorials/object-management-kubectl/declarative-object-management-configuration.md @@ -72,7 +72,7 @@ kubectl apply -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_ Print the live configuration using `kubectl get`: ```shell -kubectl get -f http://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml +kubectl get -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml ``` The output shows that the `kubectl.kubernetes.io/last-applied-configuration` annotation @@ -143,7 +143,7 @@ configuration file instead of a directory. Print the live configuration using `kubectl get`: ```shell -kubectl get -f http://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml +kubectl get -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml ``` The output shows that the `kubectl.kubernetes.io/last-applied-configuration` annotation @@ -194,7 +194,7 @@ kubectl scale deployment/nginx-deployment --replicas 2 Print the live configuration using `kubectl get`: ```shell -kubectl get -f http://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml +kubectl get -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml ``` The output shows that the `replicas` field has been set to 2, and the `last-applied-configuration` @@ -248,7 +248,7 @@ kubectl apply -f https://k8s.io/docs/tutorials/object-management-kubectl/update_ Print the live configuration using `kubectl get`: ``` -kubectl get -f http://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml +kubectl get -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml ``` The output shows the following changes to the live configuration: @@ -679,7 +679,7 @@ kubectl apply -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_ Print the live configuration using `kubectl get`: ```shell -kubectl get -f http://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml +kubectl get -f https://k8s.io/docs/tutorials/object-management-kubectl/simple_deployment.yaml -o yaml ``` The output shows that the API server set several fields to default values in the live From c959eeecd045dbaa2ab2e6089787fa3612bdafe3 Mon Sep 17 00:00:00 2001 From: chenhuan12 Date: Thu, 24 Aug 2017 15:26:01 +0800 Subject: [PATCH 18/27] fix the command output fix the command output --- docs/tasks/federation/set-up-cluster-federation-kubefed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tasks/federation/set-up-cluster-federation-kubefed.md b/docs/tasks/federation/set-up-cluster-federation-kubefed.md index 01d0da5d6ea10..c613f1e03d8d9 100644 --- a/docs/tasks/federation/set-up-cluster-federation-kubefed.md +++ b/docs/tasks/federation/set-up-cluster-federation-kubefed.md @@ -89,7 +89,7 @@ similar to the following: ``` CURRENT NAME CLUSTER AUTHINFO NAMESPACE - gke_myproject_asia-east1-b_gce-asia-east1 gke_myproject_asia-east1-b_gce-asia-east1 gke_myproject_asia-east1-b_gce-asia-east1 +* gke_myproject_asia-east1-b_gce-asia-east1 gke_myproject_asia-east1-b_gce-asia-east1 gke_myproject_asia-east1-b_gce-asia-east1 ``` From dd0c9d5c496ddb49abfe7eb67e4031710eab8f6d Mon Sep 17 00:00:00 2001 From: Alexander Prokopyev Date: Fri, 18 Aug 2017 14:32:33 +0300 Subject: [PATCH 19/27] Update dns-horizontal-autoscaler.yaml `mode` is deprecated in favor of dynamic switching starting from 1.1.2 --- docs/tasks/administer-cluster/dns-horizontal-autoscaler.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/tasks/administer-cluster/dns-horizontal-autoscaler.yaml b/docs/tasks/administer-cluster/dns-horizontal-autoscaler.yaml index a919015f3c856..f29dd2e2753f9 100644 --- a/docs/tasks/administer-cluster/dns-horizontal-autoscaler.yaml +++ b/docs/tasks/administer-cluster/dns-horizontal-autoscaler.yaml @@ -22,7 +22,6 @@ spec: - /cluster-proportional-autoscaler - --namespace=kube-system - --configmap=kube-dns-autoscaler - - --mode=linear - --target= # When cluster is using large nodes(with more cores), "coresPerReplica" should dominate. # If using small nodes, "nodesPerReplica" should dominate. From 4501e6449b426c94d11cb7752243b509d7180da7 Mon Sep 17 00:00:00 2001 From: Tim Hockin Date: Thu, 24 Aug 2017 12:50:48 -0700 Subject: [PATCH 20/27] Remove mungedocs and md-check (#5191) --- .travis.yml | 9 - _tools/mungedocs/README.md | 22 -- _tools/mungedocs/analytics.go | 58 ---- _tools/mungedocs/analytics_test.go | 94 ------ _tools/mungedocs/example_syncer.go | 121 -------- _tools/mungedocs/example_syncer_test.go | 68 ---- _tools/mungedocs/headers.go | 74 ----- _tools/mungedocs/headers_test.go | 74 ----- _tools/mungedocs/kubectl_dash_f.go | 125 -------- _tools/mungedocs/kubectl_dash_f_test.go | 143 --------- _tools/mungedocs/links.go | 238 -------------- _tools/mungedocs/links_test.go | 75 ----- _tools/mungedocs/mungedocs.go | 235 -------------- _tools/mungedocs/preformatted.go | 51 --- _tools/mungedocs/preformatted_test.go | 98 ------ _tools/mungedocs/testdata/README.md | 1 - _tools/mungedocs/testdata/example.txt | 1 - _tools/mungedocs/testdata/pod.yaml | 10 - _tools/mungedocs/testdata/test-dashes.md | 1 - _tools/mungedocs/testdata/test_underscores.md | 1 - _tools/mungedocs/toc.go | 89 ------ _tools/mungedocs/toc_test.go | 79 ----- _tools/mungedocs/util.go | 291 ------------------ _tools/mungedocs/util_test.go | 172 ----------- _tools/mungedocs/whitespace.go | 31 -- _tools/mungedocs/whitespace_test.go | 46 --- 26 files changed, 2207 deletions(-) delete mode 100644 _tools/mungedocs/README.md delete mode 100644 _tools/mungedocs/analytics.go delete mode 100644 _tools/mungedocs/analytics_test.go delete mode 100644 _tools/mungedocs/example_syncer.go delete mode 100644 _tools/mungedocs/example_syncer_test.go delete mode 100644 _tools/mungedocs/headers.go delete mode 100644 _tools/mungedocs/headers_test.go delete mode 100644 _tools/mungedocs/kubectl_dash_f.go delete mode 100644 _tools/mungedocs/kubectl_dash_f_test.go delete mode 100644 _tools/mungedocs/links.go delete mode 100644 _tools/mungedocs/links_test.go delete mode 100644 _tools/mungedocs/mungedocs.go delete mode 100644 _tools/mungedocs/preformatted.go delete mode 100644 _tools/mungedocs/preformatted_test.go delete mode 100644 _tools/mungedocs/testdata/README.md delete mode 100644 _tools/mungedocs/testdata/example.txt delete mode 100644 _tools/mungedocs/testdata/pod.yaml delete mode 100644 _tools/mungedocs/testdata/test-dashes.md delete mode 100644 _tools/mungedocs/testdata/test_underscores.md delete mode 100644 _tools/mungedocs/toc.go delete mode 100644 _tools/mungedocs/toc_test.go delete mode 100644 _tools/mungedocs/util.go delete mode 100644 _tools/mungedocs/util_test.go delete mode 100644 _tools/mungedocs/whitespace.go delete mode 100644 _tools/mungedocs/whitespace_test.go diff --git a/.travis.yml b/.travis.yml index 964b903136535..f7f8311130b81 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,6 @@ install: - cp -L -R $GOPATH/src/k8s.io/kubernetes/vendor/ $GOPATH/src/ - rm -r $GOPATH/src/k8s.io/kubernetes/vendor/ -# (2) Fetch md-check along with all its dependencies. -- git clone --depth=50 --branch=master https://github.com/kubernetes/md-check $HOME/gopath/src/k8s.io/md-check -- go get -t -v k8s.io/md-check - -# (3) Build mungedocs -- go install ./_tools/mungedocs - script: - go test -v k8s.io/kubernetes.github.io/test -- $GOPATH/bin/md-check --root-dir=$HOME/gopath/src/k8s.io/kubernetes.github.io - ./verify-docs-format.sh -- $GOPATH/bin/mungedocs --verbose --verify --upstream=origin --root-dir=$HOME/gopath/src/k8s.io/kubernetes.github.io/docs/ --repo-root=$HOME/gopath/src/k8s.io/kubernetes.github.io --skip-munges=remove-whitespace,blank-lines-surround-preformatted,header-lines,sync-examples,analytics,analytics,kubectl-dash-f,table-of-contents,md-links,kubectl-dash-f diff --git a/_tools/mungedocs/README.md b/_tools/mungedocs/README.md deleted file mode 100644 index 5fe3ed106cf45..0000000000000 --- a/_tools/mungedocs/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Documentation Mungers - -Basically this is like lint/gofmt for md docs. - -It basically does the following: -- iterate over all files in the given doc root. -- for each file split it into a slice (mungeLines) of lines (mungeLine) -- a mungeline has metadata about each line typically determined by a 'fast' regex. - - metadata contains things like 'is inside a preformatted block' - - contains a markdown header - - has a link to another file - - etc.. - - if you have a really slow regex with a lot of backtracking you might want to write a fast one to limit how often you run the slow one. -- each munger is then called in turn - - they are given the mungeLines - - they create an entirely new set of mungeLines with their modifications - - the new set is returned -- the new set is then fed into the next munger. -- in the end we might commit the end mungeLines to the file or not (--verify) - - -[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/cmd/mungedocs/README.md?pixel)]() diff --git a/_tools/mungedocs/analytics.go b/_tools/mungedocs/analytics.go deleted file mode 100644 index a7eaefa0803d4..0000000000000 --- a/_tools/mungedocs/analytics.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "strings" -) - -const analyticsMungeTag = "GENERATED_ANALYTICS" -const analyticsLinePrefix = "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/" - -func updateAnalytics(fileName string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - fileName, err := makeRepoRelative(fileName, fileName) - if err != nil { - return mlines, err - } - - link := fmt.Sprintf(analyticsLinePrefix+"%s?pixel)]()", fileName) - insertLines := getMungeLines(link) - mlines, err = removeMacroBlock(analyticsMungeTag, mlines) - if err != nil { - return mlines, err - } - - // Remove floating analytics links not surrounded by the munge tags. - for _, mline := range mlines { - if mline.preformatted || mline.header || mline.beginTag || mline.endTag { - out = append(out, mline) - continue - } - if strings.HasPrefix(mline.data, analyticsLinePrefix) { - continue - } - out = append(out, mline) - } - out = appendMacroBlock(out, analyticsMungeTag) - out, err = updateMacroBlock(out, analyticsMungeTag, insertLines) - if err != nil { - return mlines, err - } - return out, nil -} diff --git a/_tools/mungedocs/analytics_test.go b/_tools/mungedocs/analytics_test.go deleted file mode 100644 index 99306f2fe00e3..0000000000000 --- a/_tools/mungedocs/analytics_test.go +++ /dev/null @@ -1,94 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestAnalytics(t *testing.T) { - b := beginMungeTag("GENERATED_ANALYTICS") - e := endMungeTag("GENERATED_ANALYTICS") - var cases = []struct { - in string - expected string - }{ - { - "aoeu", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "prefix" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + - "\n" + "suffix", - "prefix" + "\n" + "suffix" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - { - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n", - "aoeu" + "\n" + "\n" + "\n" + - b + "\n" + - "[![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/path/to/file-name.md?pixel)]()" + "\n" + - e + "\n"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - out, err := updateAnalytics("path/to/file-name.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !expected.Equal(out) { - t.Errorf("Case %d Expected \n\n%v\n\n but got \n\n%v\n\n", i, expected.String(), out.String()) - } - } -} diff --git a/_tools/mungedocs/example_syncer.go b/_tools/mungedocs/example_syncer.go deleted file mode 100644 index c15255be43948..0000000000000 --- a/_tools/mungedocs/example_syncer.go +++ /dev/null @@ -1,121 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "io/ioutil" - "regexp" - "strings" -) - -const exampleToken = "EXAMPLE" - -const exampleLineStart = " -// -// ```yaml -// foo: -// bar: -// ``` -// -// [Download example](../../examples/guestbook/frontend-service.yaml?raw=true) -// -func syncExamples(filePath string, mlines mungeLines) (mungeLines, error) { - var err error - type exampleTag struct { - token string - linkText string - fileType string - } - exampleTags := []exampleTag{} - - // collect all example Tags - for _, mline := range mlines { - if mline.preformatted || !mline.beginTag { - continue - } - line := mline.data - if !strings.HasPrefix(line, exampleLineStart) { - continue - } - match := exampleMungeTagRE.FindStringSubmatch(line) - if len(match) < 4 { - err = fmt.Errorf("Found unparsable EXAMPLE munge line %v", line) - return mlines, err - } - tag := exampleTag{ - token: exampleToken + " " + match[1], - linkText: match[1], - fileType: match[3], - } - exampleTags = append(exampleTags, tag) - } - // update all example Tags - for _, tag := range exampleTags { - ft := "" - if tag.fileType == "json" { - ft = "json" - } - if tag.fileType == "yaml" { - ft = "yaml" - } - example, err := exampleContent(filePath, tag.linkText, ft) - if err != nil { - return mlines, err - } - mlines, err = updateMacroBlock(mlines, tag.token, example) - if err != nil { - return mlines, err - } - } - return mlines, nil -} - -// exampleContent retrieves the content of the file at linkPath -func exampleContent(filePath, linkPath, fileType string) (mungeLines, error) { - repoRel, err := makeRepoRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - fileRel, err := makeFileRelative(linkPath, filePath) - if err != nil { - return nil, err - } - - dat, err := ioutil.ReadFile(repoRel) - if err != nil { - return nil, err - } - - // remove leading and trailing spaces and newlines - trimmedFileContent := strings.TrimSpace(string(dat)) - content := fmt.Sprintf("\n```%s\n%s\n```\n\n[Download example](%s?raw=true)", fileType, trimmedFileContent, fileRel) - out := getMungeLines(content) - return out, nil -} diff --git a/_tools/mungedocs/example_syncer_test.go b/_tools/mungedocs/example_syncer_test.go deleted file mode 100644 index 3d51b28c9d819..0000000000000 --- a/_tools/mungedocs/example_syncer_test.go +++ /dev/null @@ -1,68 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func Test_syncExamples(t *testing.T) { - var podExample = `apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 -` - var textExample = `some text -` - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```yaml\n" + podExample + "```\n\n[Download example](../mungedocs/testdata/pod.yaml?raw=true)\n\n", - }, - { - "\n\n", - "\n\n```\n" + textExample + "```\n\n[Download example](testdata/example.txt?raw=true)\n\n", - }, - } - repoRoot = "" - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := syncExamples("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !expected.Equal(actual) { - t.Errorf("Expected example \n'%q' but got \n'%q'", expected.String(), actual.String()) - } - } -} diff --git a/_tools/mungedocs/headers.go b/_tools/mungedocs/headers.go deleted file mode 100644 index e23ae7536e5ee..0000000000000 --- a/_tools/mungedocs/headers.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" -) - -var headerRegex = regexp.MustCompile(`^(#+)\s*(.*)$`) - -func fixHeaderLine(mlines mungeLines, newlines mungeLines, linenum int) mungeLines { - var out mungeLines - - mline := mlines[linenum] - line := mlines[linenum].data - - matches := headerRegex.FindStringSubmatch(line) - if matches == nil { - out = append(out, mline) - return out - } - - // There must be a blank line before the # (unless first line in file) - if linenum != 0 { - newlen := len(newlines) - if newlines[newlen-1].data != "" { - out = append(out, blankMungeLine) - } - } - - // There must be a space AFTER the ##'s - newline := fmt.Sprintf("%s %s", matches[1], matches[2]) - newmline := newMungeLine(newline) - out = append(out, newmline) - - // The next line needs to be a blank line (unless last line in file) - if len(mlines) > linenum+1 && mlines[linenum+1].data != "" { - out = append(out, blankMungeLine) - } - return out -} - -// Header lines need whitespace around them and after the #s. -func updateHeaderLines(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - for i, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if !mline.header { - out = append(out, mline) - continue - } - newLines := fixHeaderLine(mlines, out, i) - out = append(out, newLines...) - } - return out, nil -} diff --git a/_tools/mungedocs/headers_test.go b/_tools/mungedocs/headers_test.go deleted file mode 100644 index d308b01034873..0000000000000 --- a/_tools/mungedocs/headers_test.go +++ /dev/null @@ -1,74 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestHeaderLines(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "# ok", - "# ok", - }, - { - "## ok", - "## ok", - }, - { - "##### ok", - "##### ok", - }, - { - "##fix", - "## fix", - }, - { - "foo\n\n##fix\n\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n##fix\nbar", - "foo\n\n## fix\n\nbar", - }, - { - "foo\n```\n##fix\n```\nbar", - "foo\n```\n##fix\n```\nbar", - }, - { - "foo\n#fix1\n##fix2\nbar", - "foo\n\n# fix1\n\n## fix2\n\nbar", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateHeaderLines("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/_tools/mungedocs/kubectl_dash_f.go b/_tools/mungedocs/kubectl_dash_f.go deleted file mode 100644 index 2dae48cc8574d..0000000000000 --- a/_tools/mungedocs/kubectl_dash_f.go +++ /dev/null @@ -1,125 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "os" - "path" - "strings" -) - -// Looks for lines that have kubectl commands with -f flags and files that -// don't exist. -func updateKubectlFileTargets(file string, mlines mungeLines) (mungeLines, error) { - var errors []string - for i, mline := range mlines { - if !mline.preformatted { - continue - } - if err := lookForKubectl(mline.data, i); err != nil { - errors = append(errors, err.Error()) - } - } - err := error(nil) - if len(errors) != 0 { - err = fmt.Errorf("%s", strings.Join(errors, "\n")) - } - return mlines, err -} - -func lookForKubectl(line string, lineNum int) error { - fields := strings.Fields(line) - for i := range fields { - if fields[i] == "kubectl" { - return gotKubectl(lineNum, fields, i) - } - } - return nil -} - -func gotKubectl(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - switch fields[i] { - case "create", "update", "replace", "delete": - return gotCommand(lineNum, fields, i) - } - } - return nil -} - -func gotCommand(lineNum int, fields []string, fieldNum int) error { - for i := fieldNum + 1; i < len(fields); i++ { - if strings.HasPrefix(fields[i], "-f") { - return gotDashF(lineNum, fields, i) - } - } - return nil -} - -func gotDashF(lineNum int, fields []string, fieldNum int) error { - target := "" - if fields[fieldNum] == "-f" { - if fieldNum+1 == len(fields) { - return fmt.Errorf("ran out of fields after '-f'") - } - target = fields[fieldNum+1] - } else { - target = fields[fieldNum][2:] - } - // Turn dirs into file-like names. - target = strings.TrimRight(target, "/") - - // Now exclude special-cases - - if target == "-" || target == "FILENAME" { - // stdin and "FILENAME" are OK - return nil - } - if strings.HasPrefix(target, "http://") || strings.HasPrefix(target, "https://") { - // URLs are ok - return nil - } - if strings.HasPrefix(target, "./") { - // Same-dir files are usually created in the same example - return nil - } - if strings.HasPrefix(target, "~/") { - // Home directory may also be created by the same example - return nil - } - if strings.HasPrefix(target, "/") { - // Absolute paths tend to be /tmp/* and created in the same example. - return nil - } - if strings.HasPrefix(target, "$") { - // Allow the start of the target to be an environment - // variable that points to the root of the kubernetes - // repo. - split := strings.SplitN(target, "/", 2) - if len(split) == 2 { - target = split[1] - } - } - - // If we got here we expect the file to exist. - _, err := os.Stat(path.Join(repoRoot, target)) - if os.IsNotExist(err) { - return fmt.Errorf("%d: target file %q does not exist", lineNum, target) - } - return err -} diff --git a/_tools/mungedocs/kubectl_dash_f_test.go b/_tools/mungedocs/kubectl_dash_f_test.go deleted file mode 100644 index 6f18fd547a408..0000000000000 --- a/_tools/mungedocs/kubectl_dash_f_test.go +++ /dev/null @@ -1,143 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "testing" - -func TestKubectlDashF(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - // No match - {"", true}, - { - "Foo\nBar\n", - true, - }, - { - "Foo\nkubectl blah blech\nBar", - true, - }, - { - "Foo\n```shell\nkubectl blah blech\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create blech\n```\nBar", - true, - }, - // Special cases - { - "Foo\n```\nkubectl -blah create -f -\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f-\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f FILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fFILENAME\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f http://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fhttp://google.com/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f ./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f./foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f /foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f/foobar\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -f~/foobar\n```\nBar", - true, - }, - // Real checks - { - "Foo\n```\nkubectl -blah create -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah create -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah update -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah replace -fmungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -f mungedocs.go\n```\nBar", - true, - }, - { - "Foo\n```\nkubectl -blah delete -fmungedocs.go\n```\nBar", - true, - }, - // Failures - { - "Foo\n```\nkubectl -blah delete -f does_not_exist\n```\nBar", - false, - }, - { - "Foo\n```\nkubectl -blah delete -fdoes_not_exist\n```\nBar", - false, - }, - } - for i, c := range cases { - repoRoot = "" - in := getMungeLines(c.in) - _, err := updateKubectlFileTargets("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success, got %v", i, err) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: unexpected success", i) - } - } -} diff --git a/_tools/mungedocs/links.go b/_tools/mungedocs/links.go deleted file mode 100644 index c05cdfa1874b7..0000000000000 --- a/_tools/mungedocs/links.go +++ /dev/null @@ -1,238 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "fmt" - "net/url" - "os" - "path" - "regexp" - "strings" -) - -var ( - // Finds markdown links of the form [foo](bar "alt-text"). - linkRE = regexp.MustCompile(`\[([^]]*)\]\(([^)]*)\)`) - // Finds markdown link typos of the form (foo)[bar] - badLinkRE = regexp.MustCompile(`\([^]()]*\)\[[^]()]*\]`) - // Splits the link target into link target and alt-text. - altTextRE = regexp.MustCompile(`([^)]*)( ".*")`) -) - -func processLink(in string, filePath string) (string, error) { - var errs []string - out := linkRE.ReplaceAllStringFunc(in, func(in string) string { - var err error - match := linkRE.FindStringSubmatch(in) - if match == nil { - errs = append(errs, fmt.Sprintf("Detected this line had a link, but unable to parse, %v", in)) - return "" - } - // match[0] is the entire expression; - visibleText := match[1] - linkText := match[2] - altText := "" - if parts := altTextRE.FindStringSubmatch(linkText); parts != nil { - linkText = parts[1] - altText = parts[2] - } - - // clean up some random garbage I found in our docs. - linkText = strings.Trim(linkText, " ") - linkText = strings.Trim(linkText, "\n") - linkText = strings.Trim(linkText, " ") - - u, terr := url.Parse(linkText) - if terr != nil { - errs = append(errs, fmt.Sprintf("link %q is unparsable: %v", linkText, terr)) - return in - } - - if u.Host != "" && u.Host != "github.com" { - // We only care about relative links and links within github. - return in - } - - suggestedVisibleText := visibleText - if u.Path != "" && !strings.HasPrefix(linkText, "TODO:") { - newPath, targetExists := checkPath(filePath, path.Clean(u.Path)) - if !targetExists { - errs = append(errs, fmt.Sprintf("%q: target not found", linkText)) - return in - } - u.Path = newPath - if strings.HasPrefix(u.Path, "/") { - u.Host = "github.com" - u.Scheme = "https" - } else { - // Remove host and scheme from relative paths - u.Host = "" - u.Scheme = "" - } - // Make the visible text show the absolute path if it's - // not nested in or beneath the current directory. - if strings.HasPrefix(u.Path, "..") { - dir := path.Dir(filePath) - suggestedVisibleText, err = makeRepoRelative(path.Join(dir, u.Path), filePath) - if err != nil { - errs = append(errs, fmt.Sprintf("%q: unable to make path relative", filePath)) - return in - } - } else { - suggestedVisibleText = u.Path - } - var unescaped string - if unescaped, err = url.QueryUnescape(u.String()); err != nil { - // Remove %28 type stuff, be nice to humans. - // And don't fight with the toc generator. - linkText = unescaped - } else { - linkText = u.String() - } - } - // If the current visible text is trying to be a file name, use - // the correct file name. - if strings.HasSuffix(visibleText, ".md") && !strings.ContainsAny(visibleText, ` '"`+"`") { - visibleText = suggestedVisibleText - } - - return fmt.Sprintf("[%s](%s)", visibleText, linkText+altText) - }) - if len(errs) != 0 { - return "", errors.New(strings.Join(errs, ",")) - } - return out, nil -} - -// updateLinks assumes lines has links in markdown syntax, and verifies that -// any relative links actually point to files that exist. -func updateLinks(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - allErrs := []string{} - - for lineNum, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - if badMatch := badLinkRE.FindString(mline.data); badMatch != "" { - allErrs = append(allErrs, - fmt.Sprintf("On line %d: found backwards markdown link %q", lineNum, badMatch)) - } - if !mline.link { - out = append(out, mline) - continue - } - line, err := processLink(mline.data, filePath) - if err != nil { - var s = fmt.Sprintf("On line %d: %s", lineNum, err.Error()) - err := errors.New(s) - allErrs = append(allErrs, err.Error()) - } - ml := newMungeLine(line) - out = append(out, ml) - } - err := error(nil) - if len(allErrs) != 0 { - err = fmt.Errorf("%s", strings.Join(allErrs, "\n")) - } - return out, err -} - -// We have to append together before path.Clean will be able to tell that stuff -// like ../docs isn't needed. -func cleanPath(dirPath, linkPath string) string { - clean := path.Clean(path.Join(dirPath, linkPath)) - if strings.HasPrefix(clean, dirPath+"/") { - out := strings.TrimPrefix(clean, dirPath+"/") - if out != linkPath { - fmt.Printf("%s -> %s\n", linkPath, out) - } - return out - } - return linkPath -} - -func checkPath(filePath, linkPath string) (newPath string, ok bool) { - dir := path.Dir(filePath) - absFilePrefixes := []string{ - "/kubernetes/kubernetes/blob/master/", - "/kubernetes/kubernetes/tree/master/", - } - for _, prefix := range absFilePrefixes { - if strings.HasPrefix(linkPath, prefix) { - linkPath = strings.TrimPrefix(linkPath, prefix) - // Now linkPath is relative to the root of the repo. The below - // loop that adds ../ at the beginning of the path should find - // the right path. - break - } - } - if strings.HasPrefix(linkPath, "/") { - // These links might go to e.g. the github issues page, or a - // file at a particular revision, or another github project - // entirely. - return linkPath, true - } - linkPath = cleanPath(dir, linkPath) - - // Fast exit if the link is already correct. - if info, err := os.Stat(path.Join(dir, linkPath)); err == nil { - if info.IsDir() { - return linkPath + "/", true - } - return linkPath, true - } - - for strings.HasPrefix(linkPath, "../") { - linkPath = strings.TrimPrefix(linkPath, "../") - } - - // Fix - vs _ automatically - nameMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return strings.Replace(s, "-", "_", -1) }, - func(s string) string { return strings.Replace(s, "_", "-", -1) }, - } - // Fix being moved into/out of admin (replace "admin" with directory - // you're doing mass movements to/from). - pathMungers := []func(string) string{ - func(s string) string { return s }, - func(s string) string { return path.Join("admin", s) }, - func(s string) string { return strings.TrimPrefix(s, "admin/") }, - } - - for _, namer := range nameMungers { - for _, pather := range pathMungers { - newPath = pather(namer(linkPath)) - for i := 0; i < 7; i++ { - // The file must exist. - target := path.Join(dir, newPath) - if info, err := os.Stat(target); err == nil { - if info.IsDir() { - return newPath + "/", true - } - return cleanPath(dir, newPath), true - } - newPath = path.Join("..", newPath) - } - } - } - return linkPath, false -} diff --git a/_tools/mungedocs/links_test.go b/_tools/mungedocs/links_test.go deleted file mode 100644 index 37f8e53171423..0000000000000 --- a/_tools/mungedocs/links_test.go +++ /dev/null @@ -1,75 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "testing" -) - -var _ = fmt.Printf - -func TestBadLinks(t *testing.T) { - var cases = []struct { - in string - }{ - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/NOTREADME.md)"}, - {"[NOTREADME](https://github.com/kubernetes/kubernetes/tree/master/docs/NOTREADME.md)"}, - {"[NOTREADME](../NOTREADME.md)"}, - } - for _, c := range cases { - in := getMungeLines(c.in) - _, err := updateLinks("filename.md", in) - if err == nil { - t.Errorf("Expected error") - } - } -} -func TestGoodLinks(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"[README](https://lwn.net)", - "[README](https://lwn.net)"}, - // _ to - - {"[README](https://github.com/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/test_dashes.md)", - "[README](../../cmd/mungedocs/testdata/test-dashes.md)"}, - // - to _ - {"[README](../../cmd/mungedocs/testdata/test-underscores.md)", - "[README](../../cmd/mungedocs/testdata/test_underscores.md)"}, - - // Does this even make sense? i dunno - {"[README](/docs/README.md)", - "[README](https://github.com/docs/README.md)"}, - {"[README](/kubernetes/kubernetes/tree/master/cmd/mungedocs/testdata/README.md)", - "[README](../../cmd/mungedocs/testdata/README.md)"}, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateLinks("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} diff --git a/_tools/mungedocs/mungedocs.go b/_tools/mungedocs/mungedocs.go deleted file mode 100644 index ef3eb2da1a5e6..0000000000000 --- a/_tools/mungedocs/mungedocs.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "errors" - "flag" - "fmt" - "io/ioutil" - "os" - "path" - "path/filepath" - "strings" -) - -var ( - verbose = flag.Bool("verbose", false, "On verification failure, emit pre-munge and post-munge versions.") - verify = flag.Bool("verify", false, "Exit with status 1 if files would have needed changes but do not change.") - norecurse = flag.Bool("norecurse", false, "Only process the files of --root-dir.") - upstream = flag.String("upstream", "upstream", "The name of the upstream Git remote to pull from") - rootDir = flag.String("root-dir", "", "Root directory containing documents to be processed.") - // "repo-root" seems like a dumb name, this is the relative path (from rootDir) to get to the repoRoot - relRoot = flag.String("repo-root", "..", `Appended to --root-dir to get the repository root. -It's done this way so that generally you just have to set --root-dir. -Examples: - * --root-dir=docs/ --repo-root=.. means the repository root is ./ - * --root-dir=/usr/local/long/path/repo/docs/ --repo-root=.. means the repository root is /usr/local/long/path/repo/ - * --root-dir=/usr/local/long/path/repo/docs/admin --repo-root=../.. means the repository root is /usr/local/long/path/repo/`) - skipMunges = flag.String("skip-munges", "", "Comma-separated list of munges to *not* run. Available munges are: "+availableMungeList) - repoRoot string - - ErrChangesNeeded = errors.New("mungedocs: changes required") - - // All of the munge operations to perform. - // TODO: allow selection from command line. (e.g., just check links in the examples directory.) - allMunges = []munge{ - // Simple "check something" functions must run first. - {"preformat-balance", checkPreformatBalance}, - // Functions which modify state. - {"remove-whitespace", updateWhitespace}, - {"table-of-contents", updateTOC}, - {"md-links", updateLinks}, - {"blank-lines-surround-preformatted", updatePreformatted}, - {"header-lines", updateHeaderLines}, - {"analytics", updateAnalytics}, - {"kubectl-dash-f", updateKubectlFileTargets}, - {"sync-examples", syncExamples}, - } - availableMungeList = func() string { - names := []string{} - for _, m := range allMunges { - names = append(names, m.name) - } - return strings.Join(names, ",") - }() -) - -// a munge processes a document, returning an updated document xor an error. -// The fn is NOT allowed to mutate 'before', if changes are needed it must copy -// data into a new byte array and return that. -type munge struct { - name string - fn func(filePath string, mlines mungeLines) (after mungeLines, err error) -} - -type fileProcessor struct { - // Which munge functions should we call? - munges []munge - - // Are we allowed to make changes? - verifyOnly bool -} - -// Either change a file or verify that it needs no changes (according to modify argument) -func (f fileProcessor) visit(path string) error { - if !strings.HasSuffix(path, ".md") { - return nil - } - - fileBytes, err := ioutil.ReadFile(path) - if err != nil { - return err - } - - mungeLines := getMungeLines(string(fileBytes)) - - modificationsMade := false - errFound := false - filePrinted := false - for _, munge := range f.munges { - after, err := munge.fn(path, mungeLines) - if err != nil || !after.Equal(mungeLines) { - if !filePrinted { - fmt.Printf("%s\n----\n", path) - filePrinted = true - } - fmt.Printf("%s:\n", munge.name) - if *verbose { - if len(mungeLines) <= 20 { - fmt.Printf("INPUT: <<<%v>>>\n", mungeLines) - fmt.Printf("MUNGED: <<<%v>>>\n", after) - } else { - fmt.Printf("not printing failed chunk: too many lines\n") - } - } - if err != nil { - fmt.Println(err) - errFound = true - } else { - fmt.Println("contents were modified") - modificationsMade = true - } - fmt.Println("") - } - mungeLines = after - } - - // Write out new file with any changes. - if modificationsMade { - if f.verifyOnly { - // We're not allowed to make changes. - return ErrChangesNeeded - } - ioutil.WriteFile(path, mungeLines.Bytes(), 0644) - } - if errFound { - return ErrChangesNeeded - } - - return nil -} - -func newWalkFunc(fp *fileProcessor, changesNeeded *bool) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - stat, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if path != *rootDir && stat.IsDir() && *norecurse { - return filepath.SkipDir - } - if err := fp.visit(path); err != nil { - *changesNeeded = true - if err != ErrChangesNeeded { - return err - } - } - return nil - } -} - -func wantedMunges() (filtered []munge) { - skipList := strings.Split(*skipMunges, ",") - skipped := map[string]bool{} - for _, m := range skipList { - if len(m) > 0 { - skipped[m] = true - } - } - for _, m := range allMunges { - if !skipped[m.name] { - filtered = append(filtered, m) - } else { - // Remove from the map so we can verify that everything - // requested was in fact valid. - delete(skipped, m.name) - } - } - if len(skipped) != 0 { - fmt.Fprintf(os.Stderr, "ERROR: requested to skip %v, but these are not valid munges. (valid: %v)\n", skipped, availableMungeList) - os.Exit(1) - } - return filtered -} - -func main() { - var err error - flag.Parse() - - if *rootDir == "" { - fmt.Fprintf(os.Stderr, "usage: %s [--help] [--verify] [--norecurse] --root-dir [--skip-munges=] [--upstream=] \n", flag.Arg(0)) - os.Exit(1) - } - - repoRoot = path.Join(*rootDir, *relRoot) - repoRoot, err = filepath.Abs(repoRoot) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - - fp := fileProcessor{ - munges: wantedMunges(), - verifyOnly: *verify, - } - - // For each markdown file under source docs root, process the doc. - // - If any error occurs: exit with failure (exit >1). - // - If verify is true: exit 0 if no changes needed, exit 1 if changes - // needed. - // - If verify is false: exit 0 if changes successfully made or no - // changes needed, exit 1 if manual changes are needed. - var changesNeeded bool - - err = filepath.Walk(*rootDir, newWalkFunc(&fp, &changesNeeded)) - if err != nil { - fmt.Fprintf(os.Stderr, "ERROR: %v\n", err) - os.Exit(2) - } - if changesNeeded { - if *verify { - fmt.Fprintf(os.Stderr, "FAIL: changes needed but not made due to --verify\n") - } else { - fmt.Fprintf(os.Stderr, "FAIL: some manual changes are still required.\n") - } - os.Exit(1) - } -} diff --git a/_tools/mungedocs/preformatted.go b/_tools/mungedocs/preformatted.go deleted file mode 100644 index 582ba981a109d..0000000000000 --- a/_tools/mungedocs/preformatted.go +++ /dev/null @@ -1,51 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import "fmt" - -// Blocks of ``` need to have blank lines on both sides or they don't look -// right in HTML. -func updatePreformatted(filePath string, mlines mungeLines) (mungeLines, error) { - var out mungeLines - inpreformat := false - for i, mline := range mlines { - if !inpreformat && mline.preformatted { - if i == 0 || out[len(out)-1].data != "" { - out = append(out, blankMungeLine) - } - // start of a preformat block - inpreformat = true - } - out = append(out, mline) - if inpreformat && !mline.preformatted { - if i >= len(mlines)-2 || mlines[i+1].data != "" { - out = append(out, blankMungeLine) - } - inpreformat = false - } - } - return out, nil -} - -// If the file ends on a preformatted line, there must have been an imbalance. -func checkPreformatBalance(filePath string, mlines mungeLines) (mungeLines, error) { - if len(mlines) > 0 && mlines[len(mlines)-1].preformatted { - return mlines, fmt.Errorf("unbalanced triple backtick delimiters") - } - return mlines, nil -} diff --git a/_tools/mungedocs/preformatted_test.go b/_tools/mungedocs/preformatted_test.go deleted file mode 100644 index ec93854ba072f..0000000000000 --- a/_tools/mungedocs/preformatted_test.go +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func TestPreformatted(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "```\nbob\n```", - "\n```\nbob\n```\n\n", - }, - { - "```\nbob\n```\n```\nnotbob\n```\n", - "\n```\nbob\n```\n\n```\nnotbob\n```\n\n", - }, - { - "```bob```\n", - "```bob```\n", - }, - { - " ```\n bob\n ```", - "\n ```\n bob\n ```\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updatePreformatted("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - - if !actual.Equal(expected) { - t.Errorf("case[%d]: expected %q got %q", i, c.expected, actual.String()) - } - } -} - -func TestPreformattedImbalance(t *testing.T) { - var cases = []struct { - in string - ok bool - }{ - {"", true}, - {"```\nin\n```", true}, - {"```\nin\n```\nout", true}, - {"```", false}, - {"```\nin\n```\nout\n```", false}, - } - for i, c := range cases { - in := getMungeLines(c.in) - out, err := checkPreformatBalance("filename.md", in) - if err != nil && c.ok { - t.Errorf("case[%d]: expected success", i) - } - if err == nil && !c.ok { - t.Errorf("case[%d]: expected failure", i) - } - // Even in case of misformat, return all the text, - // so that the user's work is not lost. - if !equalMungeLines(out, in) { - t.Errorf("case[%d]: expected munged text to be identical to input text", i) - } - } -} - -func equalMungeLines(a, b mungeLines) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} diff --git a/_tools/mungedocs/testdata/README.md b/_tools/mungedocs/testdata/README.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/_tools/mungedocs/testdata/README.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/example.txt b/_tools/mungedocs/testdata/example.txt deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/_tools/mungedocs/testdata/example.txt +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/pod.yaml b/_tools/mungedocs/testdata/pod.yaml deleted file mode 100644 index 89920b83a9af6..0000000000000 --- a/_tools/mungedocs/testdata/pod.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: nginx -spec: - containers: - - name: nginx - image: nginx - ports: - - containerPort: 80 \ No newline at end of file diff --git a/_tools/mungedocs/testdata/test-dashes.md b/_tools/mungedocs/testdata/test-dashes.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/_tools/mungedocs/testdata/test-dashes.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/testdata/test_underscores.md b/_tools/mungedocs/testdata/test_underscores.md deleted file mode 100644 index 7b57bd29ea8af..0000000000000 --- a/_tools/mungedocs/testdata/test_underscores.md +++ /dev/null @@ -1 +0,0 @@ -some text diff --git a/_tools/mungedocs/toc.go b/_tools/mungedocs/toc.go deleted file mode 100644 index 649a954bf5abc..0000000000000 --- a/_tools/mungedocs/toc.go +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "regexp" - "strings" -) - -const tocMungeTag = "GENERATED_TOC" - -var r = regexp.MustCompile("[^A-Za-z0-9-]") - -// inserts/updates a table of contents in markdown file. -// -// First, builds a ToC. -// Then, finds the magic macro block tags and replaces anything between those with -// the ToC, thereby updating any previously inserted ToC. -// -// TODO(erictune): put this in own package with tests -func updateTOC(filePath string, mlines mungeLines) (mungeLines, error) { - toc := buildTOC(mlines) - updatedMarkdown, err := updateMacroBlock(mlines, tocMungeTag, toc) - if err != nil { - return mlines, err - } - return updatedMarkdown, nil -} - -// builds table of contents for markdown file -// -// First scans for all section headers (lines that begin with "#" but not within code quotes) -// and builds a table of contents from those. Assumes bookmarks for those will be -// like #each-word-in-heading-in-lowercases-with-dashes-instead-of-spaces. -// builds the ToC. - -func buildTOC(mlines mungeLines) mungeLines { - var out mungeLines - bookmarks := map[string]int{} - - for _, mline := range mlines { - if mline.preformatted || !mline.header { - continue - } - // Add a blank line after the munge start tag - if len(out) == 0 { - out = append(out, blankMungeLine) - } - line := mline.data - noSharps := strings.TrimLeft(line, "#") - numSharps := len(line) - len(noSharps) - heading := strings.Trim(noSharps, " \n") - if numSharps > 0 { - indent := strings.Repeat(" ", numSharps-1) - bookmark := strings.Replace(strings.ToLower(heading), " ", "-", -1) - // remove symbols (except for -) in bookmarks - bookmark = r.ReplaceAllString(bookmark, "") - // Incremental counter for duplicate bookmarks - next := bookmarks[bookmark] - bookmarks[bookmark] = next + 1 - if next > 0 { - bookmark = fmt.Sprintf("%s-%d", bookmark, next) - } - tocLine := fmt.Sprintf("%s- [%s](#%s)", indent, heading, bookmark) - out = append(out, newMungeLine(tocLine)) - } - - } - // Add a blank line before the munge end tag - if len(out) != 0 { - out = append(out, blankMungeLine) - } - return out -} diff --git a/_tools/mungedocs/toc_test.go b/_tools/mungedocs/toc_test.go deleted file mode 100644 index 95ff354ed5646..0000000000000 --- a/_tools/mungedocs/toc_test.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "testing" -) - -func Test_buildTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - {"Lorem ipsum\ndolor sit amet\n", ""}, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\ndolor sit amet\n```bash\n#!/bin/sh\n```", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n", - }, - { - "# Title\nLorem ipsum \n## Section Heading\n### Ok, why doesn't this work? ...add 4 *more* `symbols`!\ndolor sit amet\n", - "\n- [Title](#title)\n - [Section Heading](#section-heading)\n - [Ok, why doesn't this work? ...add 4 *more* `symbols`!](#ok-why-doesnt-this-work-add-4-more-symbols)\n\n", - }, - } - for i, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual := buildTOC(in) - if !expected.Equal(actual) { - t.Errorf("Case[%d] Expected TOC '%v' but got '%v'", i, expected.String(), actual.String()) - } - } -} - -func Test_updateTOC(t *testing.T) { - var cases = []struct { - in string - expected string - }{ - {"", ""}, - { - "Lorem ipsum\ndolor sit amet\n", - "Lorem ipsum\ndolor sit amet\n", - }, - { - "# Title\nLorem ipsum \n**table of contents**\n\nold cruft\n\n## Section Heading\ndolor sit amet\n", - "# Title\nLorem ipsum \n**table of contents**\n\n\n- [Title](#title)\n - [Section Heading](#section-heading)\n\n\n## Section Heading\ndolor sit amet\n", - }, - } - for _, c := range cases { - in := getMungeLines(c.in) - expected := getMungeLines(c.expected) - actual, err := updateTOC("filename.md", in) - if err != nil { - t.Errorf("Error: %v", err) - } - if !expected.Equal(actual) { - t.Errorf("Expected TOC '%v' but got '%v'", expected.String(), actual.String()) - } - } -} diff --git a/_tools/mungedocs/util.go b/_tools/mungedocs/util.go deleted file mode 100644 index c25e1d1976ecc..0000000000000 --- a/_tools/mungedocs/util.go +++ /dev/null @@ -1,291 +0,0 @@ -/* -Copyright 2015 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package main - -import ( - "fmt" - "path" - "path/filepath" - "regexp" - "strings" - "unicode" -) - -// Replaces the text between matching "beginMark" and "endMark" within the -// document represented by "lines" with "insertThis". -// -// Delimiters should occupy own line. -// Returns copy of document with modifications. -func updateMacroBlock(mlines mungeLines, token string, insertThis mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted && !betweenBeginAndEnd { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - out = append(out, mline) - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - out = append(out, insertThis...) - out = append(out, mline) - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Tests that a document, represented as a slice of lines, has a line. Ignores -// leading and trailing space. -func hasLine(lines mungeLines, needle string) bool { - for _, mline := range lines { - haystack := strings.TrimSpace(mline.data) - if haystack == needle { - return true - } - } - return false -} - -func removeMacroBlock(token string, mlines mungeLines) (mungeLines, error) { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - var out mungeLines - betweenBeginAndEnd := false - for _, mline := range mlines { - if mline.preformatted { - out = append(out, mline) - continue - } - line := mline.data - if mline.beginTag && line == beginMark { - if betweenBeginAndEnd { - return nil, fmt.Errorf("found second begin mark while updating macro blocks") - } - betweenBeginAndEnd = true - } else if mline.endTag && line == endMark { - if !betweenBeginAndEnd { - return nil, fmt.Errorf("found end mark without begin mark while updating macro blocks") - } - betweenBeginAndEnd = false - } else { - if !betweenBeginAndEnd { - out = append(out, mline) - } - } - } - if betweenBeginAndEnd { - return nil, fmt.Errorf("never found closing end mark while updating macro blocks") - } - return out, nil -} - -// Add a macro block to the beginning of a set of lines -func prependMacroBlock(token string, mlines mungeLines) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mungeLines{beginLine, endLine} - if len(mlines) > 0 && mlines[0].data != "" { - out = append(out, blankMungeLine) - } - return append(out, mlines...) -} - -// Add a macro block to the end of a set of lines -func appendMacroBlock(mlines mungeLines, token string) mungeLines { - beginLine := newMungeLine(beginMungeTag(token)) - endLine := newMungeLine(endMungeTag(token)) - out := mlines - if len(mlines) > 0 && mlines[len(mlines)-1].data != "" { - out = append(out, blankMungeLine) - } - return append(out, beginLine, endLine) -} - -// Tests that a document, represented as a slice of lines, has a macro block. -func hasMacroBlock(lines mungeLines, token string) bool { - beginMark := beginMungeTag(token) - endMark := endMungeTag(token) - - foundBegin := false - for _, mline := range lines { - if mline.preformatted { - continue - } - if !mline.beginTag && !mline.endTag { - continue - } - line := mline.data - switch { - case !foundBegin && line == beginMark: - foundBegin = true - case foundBegin && line == endMark: - return true - } - } - return false -} - -// Returns the canonical begin-tag for a given description. This does not -// include the trailing newline. -func beginMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -// Returns the canonical end-tag for a given description. This does not -// include the trailing newline. -func endMungeTag(desc string) string { - return fmt.Sprintf("", desc) -} - -type mungeLine struct { - data string - preformatted bool - header bool - link bool - beginTag bool - endTag bool -} - -type mungeLines []mungeLine - -func (m1 mungeLines) Equal(m2 mungeLines) bool { - if len(m1) != len(m2) { - return false - } - for i := range m1 { - if m1[i].data != m2[i].data { - return false - } - } - return true -} - -func (mlines mungeLines) String() string { - slice := []string{} - for _, mline := range mlines { - slice = append(slice, mline.data) - } - s := strings.Join(slice, "\n") - // We need to tack on an extra newline at the end of the file - return s + "\n" -} - -func (mlines mungeLines) Bytes() []byte { - return []byte(mlines.String()) -} - -var ( - // Finds all preformatted block start/stops. - preformatRE = regexp.MustCompile("^\\s*```") - notPreformatRE = regexp.MustCompile("^\\s*```.*```") - // Is this line a header? - mlHeaderRE = regexp.MustCompile(`^#`) - // Is there a link on this line? - mlLinkRE = regexp.MustCompile(`\[[^]]*\]\([^)]*\)`) - beginTagRE = regexp.MustCompile(`