fix(rule): add memorize and concurrency limitation

package.json

@@ -45,6 +45,8 @@
     "get-url-origin": "^1.0.1",
     "minimatch": "^3.0.4",
     "node-fetch": "^2.6.0",
+    "p-all": "^2.1.0",
+    "p-memoize": "^3.1.0",
     "textlint-rule-helper": "^2.1.1"
   },
   "devDependencies": {

src/no-dead-link.js

@@ -5,12 +5,15 @@ import fs from 'fs-extra';
 import minimatch from 'minimatch';
 import { isAbsolute } from 'path';
 import { getURLOrigin } from 'get-url-origin';
+import pMemoize from 'p-memoize';
+import pAll from 'p-all';
 
 const DEFAULT_OPTIONS = {
   checkRelative: true, // {boolean} `false` disables the checks for relative URIs.
   baseURI: null, // {String|null} a base URI to resolve relative URIs.
   ignore: [], // {Array<String>} URIs to be skipped from availability checks.
   preferGET: [], // {Array<String>} origins to prefer GET over HEAD.
+  concurrency: 8 // concurrency count of linting link
 };
 
 // Adopted from http://stackoverflow.com/a/3809435/951517
@@ -23,8 +26,9 @@ const URI_REGEXP = /(?:https?:)?\/\/(?:www\.)?[-a-zA-Z0-9@:%._+~#=]{2,256}\.[a-z
  */
 function isHttp(uri) {
   const { protocol } = URL.parse(uri);
-  return protocol === "http:" || protocol === "https:"
+  return protocol === 'http:' || protocol === 'https:';
 }
+
 /**
  * Returns `true` if a given URI is relative.
  * @param {string} uri
@@ -87,7 +91,7 @@ async function isAliveURI(uri, method = 'HEAD') {
       'Accept': '*/*',
       // Same host for target url
       // https://github.com/textlint-rule/textlint-rule-no-dead-link/issues/111
-      'Host': host
+      'Host': host,
     },
     // Use `manual` redirect behaviour to get HTTP redirect status code
     // and see what kind of redirect is occurring
@@ -156,7 +160,10 @@ function reporter(context, options = {}) {
   const { Syntax, getSource, report, RuleError, fixer, getFilePath } = context;
   const helper = new RuleHelper(context);
   const opts = Object.assign({}, DEFAULT_OPTIONS, options);
-
+  // 30sec cache
+  const memorizedIsAliveURI = pMemoize(isAliveURI, {
+    maxAge: 30 * 1000,
+  });
   /**
    * Checks a given URI's availability and report if it is dead.
    * @param {TextLintNode} node TextLintNode the URI belongs to.
@@ -202,7 +209,7 @@ function reporter(context, options = {}) {
 
     const result = isLocal(uri)
       ? await isAliveLocalFile(uri)
-      : await isAliveURI(uri, method);
+      : await memorizedIsAliveURI(uri, method);
     const { ok, redirected, redirectTo, message } = result;
 
     if (!ok) {
@@ -269,7 +276,10 @@ function reporter(context, options = {}) {
     },
 
     [`${context.Syntax.Document}:exit`]() {
-      return Promise.all(URIs.map((item) => lint(item)));
+      const linkTasks = URIs.map((item) => () => lint(item));
+      return pAll(linkTasks, {
+        concurrency: opts.concurrency
+      });
     },
   };
 }

test/no-dead-link.js

@@ -12,6 +12,8 @@ tester.run('no-dead-link', rule, {
     'should ignore non-http url [ftp](ftp://example.com) by default',
     'should ignore non-http url [websockets](ws://example.com) by default',
     'should be able to check a link in Markdown: [example](https://example.com/)',
+    'should be able to check a link in Markdown: [example](https://jcp.org/en/jsr/detail?id=330)',
+    'should be able to check a link in Markdown: [example](https://dev.mysql.com/downloads/mysql/)',
     'should be able to check a URL in Markdown: https://example.com/',
     'should success with retrying on error: [npm results for textlint](https://www.npmjs.com/search?q=textlint)',
     'should treat 200 OK as alive: https://httpstat.us/200',

yarn.lock

@@ -3265,7 +3265,7 @@ md5@^2.2.1:
     crypt "~0.0.1"
     is-buffer "~1.1.1"
 
-mem@^4.0.0:
+mem@^4.0.0, mem@^4.3.0:
   version "4.3.0"
   resolved "https://registry.yarnpkg.com/mem/-/mem-4.3.0.tgz#461af497bc4ae09608cdb2e60eefb69bff744178"
   integrity sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==
@@ -3317,7 +3317,7 @@ mimic-fn@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-1.1.0.tgz#e667783d92e89dbd342818b5230b9d62a672ad18"
 
-mimic-fn@^2.0.0:
+mimic-fn@^2.0.0, mimic-fn@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b"
   integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==
@@ -3717,6 +3717,13 @@ output-file-sync@^2.0.0:
     is-plain-obj "^1.1.0"
     mkdirp "^0.5.1"
 
+p-all@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/p-all/-/p-all-2.1.0.tgz#91419be56b7dee8fe4c5db875d55e0da084244a0"
+  integrity sha512-HbZxz5FONzz/z2gJfk6bFca0BCiSRF8jU3yCsWOen/vR6lZjfPOu/e7L3uFzTW1i0H8TlC3vqQstEJPQL4/uLA==
+  dependencies:
+    p-map "^2.0.0"
+
 p-defer@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c"
@@ -3771,6 +3778,14 @@ p-map@^2.0.0:
   resolved "https://registry.yarnpkg.com/p-map/-/p-map-2.1.0.tgz#310928feef9c9ecc65b68b17693018a665cea175"
   integrity sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==
 
+p-memoize@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/p-memoize/-/p-memoize-3.1.0.tgz#ac7587983c9e530139f969ca7b41ef40e93659aa"
+  integrity sha512-e5tIvrsr7ydUUnxb534iQWtXxWgk/86IsH+H+nV4FHouIggBt4coXboKBt26o4lTu7JbEnGSeXdEsYR8BhAHFA==
+  dependencies:
+    mem "^4.3.0"
+    mimic-fn "^2.1.0"
+
 p-try@^2.0.0:
   version "2.2.0"
   resolved "https://registry.yarnpkg.com/p-try/-/p-try-2.2.0.tgz#cb2868540e313d61de58fafbe35ce9004d5540e6"