Update markdown lint and linkchecker (#521)

* update CI and scripts Signed-off-by: Nicolas MASSART <[email protected]> * fix MD and ignore log files Signed-off-by: Nicolas MASSART <[email protected]> * remove previously commited logs Signed-off-by: Nicolas MASSART <[email protected]> * fix package path Signed-off-by: Nicolas MASSART <[email protected]> * update link checker conf Signed-off-by: Nicolas MASSART <[email protected]> * restored some link exclusions probably ddos protection on etherescan and internal postman link is in an include making path wrong when checked separately Signed-off-by: Nicolas MASSART <[email protected]>
hyperledger · Sep 24, 2020 · 82e8ca4 · 82e8ca4
1 parent a5461b0
commit 82e8ca4
Show file tree

Hide file tree

Showing 62 changed files with 684 additions and 269 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -10,9 +10,6 @@ executors:
   shell_executor:
     docker:
       - image: circleci/buildpack-deps:buster-scm
-  ruby_executor:
-    docker:
-      - image: circleci/ruby:2-stretch
 
 jobs:
   dco:
@@ -50,7 +47,7 @@ jobs:
           command: |
             python3 -m venv venv
             . venv/bin/activate
-            pip install -r CI/requirements.txt
+            pip install -q -r CI/requirements.txt
       - save_cache:
           paths:
             - ./venv
@@ -86,40 +83,47 @@ jobs:
           command: 'sudo npm install -g npm@latest'
       - restore_cache:
           keys:
-            - deps-{{ checksum "package.json" }}
+            - deps-{{ checksum "package-lock.json" }}
       - run:
           name: Install dependencies
           command: |
-            npm install
+            npm ci
       - run:
           name: Run markdown link checker
           command: |
             npm run test:links
       - save_cache:
           paths:
             - ./node_modules
-          key: deps-{{ checksum "package.json" }}
+          key: deps-{{ checksum "package-lock.json" }}
       - store_artifacts:
           path: ./linkchecker.out
           destination: linkchecker.out
 
   markdownlint:
-    executor: ruby_executor
+    executor: node_executor
     steps:
       - checkout
+      - restore_cache:
+          keys:
+            - deps-{{ checksum "package-lock.json" }}
       - run:
           name: Install dependencies
           command: |
-            gem install mdl
+            npm ci
       - run:
           shell: /bin/bash #this is a non breaking command so it will always return success
           name: Run Markdownlint info checks
           command: |
-            mdl --ignore-front-matter --style ./CI/markdownlint/info_style.rb . | tee ./markdownlint_info.out
+            npm run test:markdown:info
       - run:
           name: Run Markdownlint
           command: |
-            mdl --ignore-front-matter --style ./CI/markdownlint/style.rb . | tee ./markdownlint.out
+            npm run test:markdown
+      - save_cache:
+          paths:
+            - ./node_modules
+          key: deps-{{ checksum "package-lock.json" }}
       - store_artifacts:
           path: ./markdownlint.out
           destination: ./markdownlint.out

diff --git a/.gitignore b/.gitignore
@@ -5,3 +5,4 @@ env/
 node_modules/
 .circleci/process.yml
 .DS_Store
+*.log
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,2 +1,2 @@
 For changes see the Hyperledger Besu
-[CHANGELOG.md](https://github.com/hyperledger/besu/blob/master/CHANGELOG.md)
+[CHANGELOG.md](https://github.com/hyperledger/besu/blob/master/CHANGELOG.md)
diff --git a/CI/linkchecker/link_check_conf.json b/CI/linkchecker/link_check_conf.json
@@ -7,22 +7,12 @@
             "pattern": "^http(s)?://127.0.0.1"
         },
         {
-            "pattern": "^http(s)?://.+.etherscan.io/txs\\?block="
-        },
-        {
-            "pattern": "^http(s)?://.+.etherscan.io/tx/0x"
-        },
-        {
-            "pattern": "^http(s)?://.+.etherscan.io/address/0x"
-        },
-        {
-            "pattern": "^http(s)?://.+.blockscout.com/"
+            "pattern": "^http(s)?://.+.etherscan.io/"
         },
         {
             "pattern": "/postman/.+.json"
-        },
-        {
-            "pattern": "https://remix.ethereum.org/"
         }
-    ]
+    ],
+    "aliveStatusCodes":[200, 206],
+    "retryOn429":true
 }
diff --git a/CI/markdownlint/info_style.rb b/CI/markdownlint/info_style.rb
diff --git a/CI/markdownlint/lint-base-style.yml b/CI/markdownlint/lint-base-style.yml
@@ -0,0 +1,23 @@
+default: true
+no-trailing-punctuation:
+  punctuation: '.,;:!'
+ul-indent:
+  indent: 4
+
+no-bare-urls: false
+code-block-style: false
+line-length: false
+
+  # excluded rule for as kramdown has a bug.
+  # see https://github.com/markdownlint/markdownlint/issues/294#issuecomment-600600407
+  # these are now in info style.
+  # to be put back in error style once bug fixed.
+single-h1: false
+no-space-in-code: false
+no-duplicate-header: false
+first-line-h1: false
+
+  # Because I hate HTML but sometimes a table is hard to format in MD
+  # now in info style.
+no-inline-html: false
+
diff --git a/CI/markdownlint/lint-info-style.yml b/CI/markdownlint/lint-info-style.yml
@@ -0,0 +1,20 @@
+line-length:
+  line_length: 100
+  code_blocks: false
+  tables: false
+
+no-bare-urls: false
+
+  # excluded rule for error as kramdown has a bug.
+  # see https://github.com/markdownlint/markdownlint/issues/294#issuecomment-600600407
+  # these are now in this info style.
+  # to be put back in error style once bug fixed.
+single-h1: true
+no-space-in-code: true
+no-duplicate-header:
+  allow_different_nesting: true
+first-line-h1: true
+
+  # Because I hate HTML but sometimes a table is hard to format in MD
+no-inline-html: true
+
diff --git a/CI/markdownlint/style.rb b/CI/markdownlint/style.rb
diff --git a/CI/scripts/README.md b/CI/scripts/README.md
@@ -17,7 +17,7 @@ Go to the besu-doc project root directory and run one of the following scripts:
 * `CI/scripts/test_links.sh` tests the internal and external links in the doc. If a link is
   incorrect or the targeted web page is unavailable (for external sites), the test will fail and
   display the faulty link.
-* `CI/scripts/test_lint.sh` tests the Markdown syntax for issues. Sometimes they are not visible
+* `CI/scripts/test_markdown_syntax.sh` tests the Markdown syntax for issues. Sometimes they are not visible
   but making sure the markdown is correct helps to make it readable and bug free.
 * `CI/scripts/test_all.sh` runs all the tests in one pass.
 

diff --git a/CI/scripts/run_job.sh b/CI/scripts/run_job.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+set -o pipefail
+
+if [ -z "$1" ]
+  then
+    echo "No job mame supplied. See .circleci/config.yml for job names."
+    exit 1
+  else
+    JOB=$1
+fi
+
+exec < /dev/tty
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+LOGFILE=${DIR}/${JOB}.log
+
+echo -e "\rRunning ${JOB} test job, please wait.\r"
+
+circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job $JOB > ${LOGFILE} 2>&1
+
+if [ "$?" -eq "0" ]
+then
+  echo -e "\r✓ ${JOB} test job succeeded\r"
+  exit 0
+else
+  echo -e "\r✖ ${JOB} test job failed. See ${LOGFILE}"
+  exit 1
+fi
diff --git a/CI/scripts/test_all.sh b/CI/scripts/test_all.sh
@@ -1,5 +1,16 @@
 #!/usr/bin/env bash
-CI/scripts/test_build.sh
-CI/scripts/test_guidelines.sh
-CI/scripts/test_links.sh
-CI/scripts/test_lint.sh
+
+echo -e "\r================================="
+echo -e "\rRunning all tests jobs, please wait."
+echo -e "\r================================="
+
+CI/scripts/run_job.sh build & buildPID=$!
+CI/scripts/run_job.sh vale & guidelinesPID=$!
+CI/scripts/run_job.sh linkchecker & linksPID=$!
+CI/scripts/run_job.sh markdownlint & syntaxPID=$!
+
+wait $buildPID $linksPID $syntaxPID $guidelinesPID
+
+echo -e "\r================================="
+echo -e "\rAll tests jobs completed."
+echo -e "\r================================="
diff --git a/CI/scripts/test_build.sh b/CI/scripts/test_build.sh
@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
-circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job build
+
+CI/scripts/run_job.sh build
diff --git a/CI/scripts/test_guidelines.sh b/CI/scripts/test_guidelines.sh
@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
-circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job vale
+
+CI/scripts/run_job.sh vale
diff --git a/CI/scripts/test_links.sh b/CI/scripts/test_links.sh
@@ -1,2 +1,3 @@
 #!/usr/bin/env bash
-circleci config process .circleci/config.yml > .circleci/process.yml && circleci local execute -c .circleci/process.yml --job linkchecker
+
+CI/scripts/run_job.sh linkchecker
diff --git a/CI/scripts/test_lint.sh b/CI/scripts/test_lint.sh
diff --git a/CI/scripts/test_markdown_syntax.sh b/CI/scripts/test_markdown_syntax.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+CI/scripts/run_job.sh markdownlint
diff --git a/SECURITY.md b/SECURITY.md
@@ -17,4 +17,4 @@ The other way is to file a confidential security bug in our
 
 The process by which the Hyperledger Security Team handles security bugs is documented further in
 our [Defect Response page](https://wiki.hyperledger.org/display/SEC/Defect+Response) on our
-[wiki](https://wiki.hyperledger.org).
+[wiki](https://wiki.hyperledger.org).
diff --git a/docs/Concepts/Consensus-Protocols/Comparing-PoA.md b/docs/Concepts/Consensus-Protocols/Comparing-PoA.md
@@ -69,4 +69,4 @@ operating in adversarial environments.
 Reaching consensus and adding blocks is faster in Clique networks. For Clique, the probability of a
 fork increases as the number of validators increases.
 
-For IBFT 2.0, the time to add new blocks increases as the number of validators increases.
+For IBFT 2.0, the time to add new blocks increases as the number of validators increases.
diff --git a/docs/Concepts/Mining.md b/docs/Concepts/Mining.md
@@ -14,4 +14,4 @@ Ethminer has been used with Hyperledger Besu to mine blocks on the [Ropsten test
 [ETC mainnet (uncle block only)](https://etc.tokenview.com/en/uncleblock/10555173) and Mordor ETC testnet.
 
 !!! note
-    Some mining software supports the `getwork` scheme as the `http` scheme.
+    Some mining software supports the `getwork` scheme as the `http` scheme.
diff --git a/docs/Concepts/Monitoring.md b/docs/Concepts/Monitoring.md
@@ -12,4 +12,3 @@ logging enables:
 
 For an overview of monitoring Hyperledger Besu, view
 [this recording](https://www.youtube.com/watch?v=7BuutRe0I28&feature=youtu.be).
-
diff --git a/docs/Concepts/Network-vs-Node.md b/docs/Concepts/Network-vs-Node.md
@@ -13,4 +13,4 @@ include `evmStackSize` or specify the
 Specify node settings on the command line or in the
 [node configuration file](../HowTo/Configure/Using-Configuration-File.md). For example, enable
 [JSON-RPC API methods](../Reference/API-Methods.md) or specify the
-[data directory](../Reference/CLI/CLI-Syntax.md#data-path) for the node.
+[data directory](../Reference/CLI/CLI-Syntax.md#data-path) for the node.
diff --git a/docs/Concepts/Plugins.md b/docs/Concepts/Plugins.md
@@ -48,4 +48,4 @@ Each plugin in the directory has the following lifecycle events:
 
 !!! note
 
-    The order in which Besu calls plugins during lifecycle events is not guaranteed.
+    The order in which Besu calls plugins during lifecycle events is not guaranteed.
diff --git a/docs/Concepts/Privacy/Private-Transactions.md b/docs/Concepts/Privacy/Private-Transactions.md
@@ -82,4 +82,3 @@ for the private transaction with the incorrect nonce.
 <!-- links ---->
 
 [privacy marker transaction]: ../../Concepts/Privacy/Private-Transaction-Processing.md
-
diff --git a/docs/Concepts/Protocol-Upgrades.md b/docs/Concepts/Protocol-Upgrades.md
@@ -27,4 +27,4 @@ example, [Istanbul](https://eips.ethereum.org/EIPS/eip-1679).
     For compatibility with future protocol upgrades, do not hardcode any gas price assumptions.
 
     Implementing upgradeable contracts enables contracts to be upgraded if a protocol upgrade does
-    include breaking changes.
+    include breaking changes.
diff --git a/docs/Concepts/TLS.md b/docs/Concepts/TLS.md
@@ -12,4 +12,4 @@ or server ([Orion](https://docs.orion.pegasys.tech/en/latest/Concepts/TLS-Commun
 
 You must store private keys and certificates in password-protected PKCS #12 keystore files.
 
-Use the command line options to [enable and configure](../HowTo/Configure/Configure-TLS.md) TLS.
+Use the command line options to [enable and configure](../HowTo/Configure/Configure-TLS.md) TLS.
diff --git a/docs/HowTo/Backup/Backup.md b/docs/HowTo/Backup/Backup.md
@@ -52,4 +52,4 @@ The process for finding peers after restarting is the same as for
 [finding peers after upgrading and restarting].
 
 <!-- Links -->
-[finding peers after upgrading and restarting]: ../Upgrade/Upgrade-Node.md#finding-peers-on-restarting
+[finding peers after upgrading and restarting]: ../Upgrade/Upgrade-Node.md#finding-peers-on-restarting
diff --git a/docs/HowTo/Configure/Configure-Mining.md b/docs/HowTo/Configure/Configure-Mining.md
@@ -74,4 +74,4 @@ have been used to mine on public networks with Hyperledger Besu:
 
 * **Ropsten**: [`0x2f14582947E292a2eCd20C430B46f2d27CFE213c`](https://ropsten.etherscan.io/address/0x2f14582947E292a2eCd20C430B46f2d27CFE213c#mine)
 * **ETC**: [`0x3125309aa670f5e60493b50884a7e7abf9ebb701`](https://etc.tokenview.com/en/address/0x3125309aa670f5e60493b50884a7e7abf9ebb701)
-* **Mordor**: `0x2f14582947E292a2eCd20C430B46f2d27CFE213c`
+* **Mordor**: `0x2f14582947E292a2eCd20C430B46f2d27CFE213c`
diff --git a/docs/HowTo/Configure/Configure-TLS.md b/docs/HowTo/Configure/Configure-TLS.md
@@ -130,4 +130,4 @@ The command line:
 
 <!-- Links -->
 [Configure the client for TLS]: https://docs.ethsigner.pegasys.tech/en/latest/HowTo/Configure-TLS/#server-tls-connection
-[Configure the server to allow TLS communication]: https://docs.orion.pegasys.tech/en/latest/Tutorials/TLS/
+[Configure the server to allow TLS communication]: https://docs.orion.pegasys.tech/en/latest/Tutorials/TLS/
diff --git a/docs/HowTo/Configure/Consensus-Protocols/Clique.md b/docs/HowTo/Configure/Consensus-Protocols/Clique.md
@@ -159,4 +159,4 @@ Define the number of blocks between epoch transitions in the genesis file.
 
 <!-- Acronyms and Definitions -->
 
-*[vanity data]: Signers can include anything they like as vanity data.
+*[vanity data]: Signers can include anything they like as vanity data.
diff --git a/docs/HowTo/Configure/Consensus-Protocols/QuorumIBFT.md b/docs/HowTo/Configure/Consensus-Protocols/QuorumIBFT.md
@@ -21,4 +21,4 @@ To connect to a Quorum IBFT 1.0 network:
     ```
 
 1. When starting Besu, specify the IBFT 1.0 genesis file using the
-   [`--genesis-file`](../../../Reference/CLI/CLI-Syntax.md#genesis-file) option.
+   [`--genesis-file`](../../../Reference/CLI/CLI-Syntax.md#genesis-file) option.
diff --git a/docs/HowTo/Configure/Contracts-in-Genesis.md b/docs/HowTo/Configure/Contracts-in-Genesis.md
@@ -30,4 +30,4 @@ The contract code in the genesis file defines the:
 * Address
 * Balance
 * Bytecode
-* Key value pairs for contract storage.
+* Key value pairs for contract storage.
diff --git a/docs/HowTo/Configure/Passing-JVM-Options.md b/docs/HowTo/Configure/Passing-JVM-Options.md
@@ -18,4 +18,4 @@ setting it before starting Besu.
     ```bash
     BESU_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
     besu --network=rinkeby
-    ```
+    ```
diff --git a/docs/HowTo/Deploy/Ansible.md b/docs/HowTo/Deploy/Ansible.md
@@ -14,4 +14,4 @@ For more information, see the
 !!! important
 
     We strongly recommend automating network creation. Automating makes updates easier and ensures
-    your configuration is synchronized across the network.
+    your configuration is synchronized across the network.
diff --git a/docs/HowTo/Deploy/Bootnodes.md b/docs/HowTo/Deploy/Bootnodes.md
@@ -54,4 +54,4 @@ command line option for each node to include the new bootnodes.
 When adding bootnodes, you do not need to restart running nodes. By updating the
 [`--bootnodes`](../../Reference/CLI/CLI-Syntax.md#bootnodes) option, the next time you restart the
 nodes (for example, when [upgrading](../Upgrade/Upgrade-Node.md)), the nodes connect to the new
-bootnodes.
+bootnodes.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		#!/usr/bin/env bash

		CI/scripts/run_job.sh markdownlint
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,3 @@ logging enables:

		For an overview of monitoring Hyperledger Besu, view
		[this recording](https://www.youtube.com/watch?v=7BuutRe0I28&feature=youtu.be).
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,4 +48,4 @@ Each plugin in the directory has the following lifecycle events:

		!!! note

		The order in which Besu calls plugins during lifecycle events is not guaranteed.
		The order in which Besu calls plugins during lifecycle events is not guaranteed.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -82,4 +82,3 @@ for the private transaction with the incorrect nonce.
		<!-- links ---->

		[privacy marker transaction]: ../../Concepts/Privacy/Private-Transaction-Processing.md
Original file line number	Diff line number	Diff line change
Expand Up		@@ -12,4 +12,4 @@ or server ([Orion](https://docs.orion.pegasys.tech/en/latest/Concepts/TLS-Commun

		You must store private keys and certificates in password-protected PKCS #12 keystore files.

		Use the command line options to [enable and configure](../HowTo/Configure/Configure-TLS.md) TLS.
		Use the command line options to [enable and configure](../HowTo/Configure/Configure-TLS.md) TLS.
Original file line number	Diff line number	Diff line change
Expand Up		@@ -159,4 +159,4 @@ Define the number of blocks between epoch transitions in the genesis file.

		<!-- Acronyms and Definitions -->

		*[vanity data]: Signers can include anything they like as vanity data.
		*[vanity data]: Signers can include anything they like as vanity data.
-Original file line number
+Diff line change
@@ Expand Up / @@ -18,4 +18,4 @@ setting it before starting Besu. @@
         ```bash
         BESU_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5005 \
         besu --network=rinkeby
-        ```
+        ```