Mark removed types as deprecated

.github/workflows/ci.yml

@@ -51,6 +51,11 @@ jobs:
     uses: ./.github/workflows/retag.yml
     with:
       dry-run: true
+  deprecate-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/deprecate.yml
+    with:
+      dry-run: true
   publish_alpha:
     name: publish alpha release
     runs-on: ubuntu-latest

.github/workflows/deprecate.yml

@@ -0,0 +1,37 @@
+name: Mark removed types as deprecated
+#description: Loop over npm @types packages and mark as deprecated any that no longer exist in the DT repo.
+on:
+  schedule:
+    # https://crontab.guru/#0_0_*_*_0
+    - cron: 0 0 * * 0
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  deprecate:
+    if: github.event_name != 'schedule' || github.repository == 'microsoft/DefinitelyTyped-tools'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Parse declarations
+        run: yarn workspace @definitelytyped/publisher parse
+      - name: Mark removed types as deprecated${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: node --require source-map-support/register packages/deprecate/${{ (inputs || github.event.inputs).dry-run && ' --dry-run' || '' }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/

README.md

@@ -3,6 +3,7 @@
 A monorepo for formerly disparate DefinitelyTyped-related tools:
 
 - [definitions-parser](packages/definitions-parser): the part of [microsoft/types-publisher](https://github.com/microsoft/types-publisher) that reads DefinitelyTyped repository data
+- [deprecate](packages/publish-registry): Loop over npm @types packages and mark as deprecated any that no longer exist in the DT repo.
 - [dtslint](packages/dtslint): [microsoft/dtslint](https://github.com/microsoft/dtslint)
 - [dtslint-runner](packages/dtslint-runner): [DefinitelyTyped/dtslint-runner](https://github.com/DefinitelyTyped/dtslint-runner)
 - [dts-critic](packages/dts-critic): [DefinitelyTyped/dts-critic](https://github.com/DefinitelyTyped/dts-critic)

packages/deprecate/README.md

@@ -0,0 +1,55 @@
+# deprecate
+
+[![Mark removed types as deprecated](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/deprecate.yml/badge.svg)](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/deprecate.yml)
+
+Loop over npm @types packages and mark as deprecated any that no longer exist in the DT repo.
+
+## Use
+
+```sh
+yarn workspace @definitelytyped/publisher parse
+node packages/deprecate/
+```
+
+1. [Parse declarations](../publisher/README.md#parse-the-definitions).
+2. Run this package's script.
+
+### Options
+
+<dl><dt>
+
+`--dry-run`
+
+</dt><dd>
+
+Don't actually mark anything as deprecated, just show what would be done.
+
+</dd></dl>
+
+### Environment variables
+
+<dl><dt>
+
+`GITHUB_TOKEN`
+
+</dt><dd>
+
+Required.
+Used to talk to [GitHub's GraphQL API](https://docs.github.com/en/graphql/guides/forming-calls-with-graphql#authenticating-with-graphql), to find the commit/PR that removed the types.
+That data is public and a GitHub Actions [automatic token](https://docs.github.com/en/actions/security-guides/automatic-token-authentication) is sufficient.
+
+</dd><dt>
+
+[`NPM_TOKEN`](https://docs.npmjs.com/about-access-tokens)
+
+</dt><dd>
+
+Not required for a dry run.
+Only used to actually mark @types packages as deprecated.
+
+</dd></dl>
+
+## Logs
+
+GitHub Actions runs this package's script weekly.
+You can [examine the logs](https://github.com/microsoft/DefinitelyTyped-tools/actions/workflows/deprecate.yml).

packages/deprecate/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "@definitelytyped/deprecate",
+  "version": "1.0.0",
+  "description": "Loop over npm @types packages and mark as deprecated any that no longer exist in the DT repo.",
+  "license": "MIT",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsc --build"
+  },
+  "dependencies": {
+    "@definitelytyped/definitions-parser": "^0.0.119",
+    "@definitelytyped/utils": "^0.0.119",
+    "@octokit/graphql": "^4.8.0",
+    "libnpmsearch": "^5.0.3",
+    "yargs": "^17.5.1"
+  },
+  "devDependencies": {
+    "@types/libnpmsearch": "^2.0.3",
+    "@types/yargs": "^17.0.10"
+  }
+}

packages/deprecate/src/index.ts

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+
+import console from "console";
+import process from "process";
+import { AllPackages, getDefinitelyTyped } from "@definitelytyped/definitions-parser";
+import { NpmPublishClient } from "@definitelytyped/utils";
+import { graphql } from "@octokit/graphql";
+import search from "libnpmsearch";
+import yargs from "yargs";
+
+main();
+
+async function main() {
+  const { dryRun } = yargs.argv as never;
+  const options = { definitelyTypedPath: undefined, progress: false, parseInParallel: false };
+  const dt = await getDefinitelyTyped(options, console);
+  const allPackages = await AllPackages.read(dt);
+  const client = await NpmPublishClient.create(process.env.NPM_TOKEN!);
+  // Loop over npm @types packages and mark as deprecated any that no longer exist in the DT repo.
+  let from = 0;
+  let results;
+  do {
+    const opts = { limit: 250, from };
+    // Won't return already-deprecated packages.
+    results = await search("@types", opts);
+    for (const result of results) {
+      const types = result.name.slice("@types/".length);
+      // Skip ones that exist, either in the types/ directory or notNeededPackages.json.
+      if (allPackages.tryGetLatestVersion(types) || allPackages.getNotNeededPackage(types)) continue;
+      const msg = await fetchMsg(types);
+      if (!msg) {
+        console.log(`Could not find the commit that removed types/${types}/.`);
+        continue;
+      }
+      console.log(`Deprecating ${result.name}: ${msg}`);
+      if (!dryRun) await client.deprecate(result.name, "*", msg);
+    }
+    from += results.length;
+    // The registry API clamps limit at 250 and from at 5,000, so we can only loop over 5,250 packages, for now.
+  } while (results.length >= 250 && from <= 5000);
+}
+
+/** Reference the commit/PR that removed the named types. */
+async function fetchMsg(types: string) {
+  const {
+    repository: {
+      defaultBranchRef: {
+        target: {
+          history: {
+            nodes: [commit],
+          },
+        },
+      },
+    },
+  } = await graphql(
+    `
+      query ($path: String!) {
+        repository(name: "DefinitelyTyped", owner: "DefinitelyTyped") {
+          defaultBranchRef {
+            target {
+              ... on Commit {
+                history(first: 1, path: $path) {
+                  nodes {
+                    associatedPullRequests(first: 1) {
+                      nodes {
+                        url
+                      }
+                    }
+                    messageHeadline
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    `,
+    {
+      headers: { authorization: `token ${process.env.GITHUB_TOKEN}` },
+      path: `types/${types}/`,
+    }
+  );
+  if (!commit) return;
+  const {
+    associatedPullRequests: {
+      nodes: [pullRequest],
+    },
+    messageHeadline,
+  } = commit;
+  const subject = messageHeadline.replace(new RegExp(String.raw`^\[${types}] `), "").replace(/ \(#[0-9]+\)$/, "");
+  return pullRequest ? `${subject} ${pullRequest.url}` : subject;
+}

packages/deprecate/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "rootDir": "src/",
+    "outDir": "dist/"
+  },
+  "references": [{ "path": "../definitions-parser/" }, { "path": "../utils/" }]
+}

tsconfig.json

@@ -3,14 +3,15 @@
   "files": [],
   "references": [
     { "path": "packages/definitions-parser" },
+    { "path": "packages/deprecate" },
     { "path": "packages/dts-critic" },
     { "path": "packages/dtslint" },
     { "path": "packages/dtslint-runner" },
     { "path": "packages/header-parser" },
     { "path": "packages/perf" },
     { "path": "packages/publisher" },
+    { "path": "packages/retag" },
     { "path": "packages/typescript-versions" },
-    { "path": "packages/utils" },
-    { "path": "packages/retag" }
+    { "path": "packages/utils" }
   ]
 }

yarn.lock

@@ -1563,7 +1563,7 @@
     is-plain-object "^3.0.0"
     universal-user-agent "^5.0.0"
 
-"@octokit/graphql@^4.5.8":
+"@octokit/graphql@^4.5.8", "@octokit/graphql@^4.8.0":
   version "4.8.0"
   resolved "https://registry.yarnpkg.com/@octokit/graphql/-/graphql-4.8.0.tgz#664d9b11c0e12112cbf78e10f49a05959aa22cc3"
   integrity sha512-0gv+qLSBLKF0z8TKaSKTsS39scVKF9dbMxJpj3U0vC7wjNWFuIpL/z76Qe2fiuCbDRcJSavkXsVtMS6/dtQQsg==
@@ -1909,6 +1909,14 @@
   resolved "https://registry.yarnpkg.com/@types/json-stable-stringify/-/json-stable-stringify-1.0.33.tgz#099b0712d824d15e2660c20e1c16e6a8381f308c"
   integrity sha512-qEWiQff6q2tA5gcJGWwzplQcXdJtm+0oy6IHGHzlOf3eFAkGE/FIPXZK9ofWgNSHVp8AFFI33PJJshS0ei3Gvw==
 
+"@types/libnpmsearch@^2.0.3":
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/@types/libnpmsearch/-/libnpmsearch-2.0.3.tgz#6a7bba71e533d5344cd04ceac4fe81b6b1ab9ceb"
+  integrity sha512-f/tTUDiOaUNk+m1mfvXO4/7ZasYUaKdosLgvzMdrFHNFqJENqwT9kKE+Gd6N3nsoD5kCZ7q4Pw7ApPIjXQArbA==
+  dependencies:
+    "@types/node" "*"
+    "@types/npm-registry-fetch" "*"
+
 "@types/minimatch@*", "@types/minimatch@^3.0.3":
   version "3.0.5"
   resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.5.tgz#1001cc5e6a3704b83c236027e77f2f58ea010f40"
@@ -2124,6 +2132,13 @@
   dependencies:
     "@types/yargs-parser" "*"
 
+"@types/yargs@^17.0.10":
+  version "17.0.10"
+  resolved "https://registry.yarnpkg.com/@types/yargs/-/yargs-17.0.10.tgz#591522fce85d8739bca7b8bb90d048e4478d186a"
+  integrity sha512-gmEaFwpj/7f/ROdtIlci1R1VYU1J4j95m8T+Tj3iBgiBFKg1foE/PSl93bBd5T9LDXNPo8UlNN6W0qwD8O5OaA==
+  dependencies:
+    "@types/yargs-parser" "*"
+
 "@typescript-eslint/eslint-plugin@^4.8.1":
   version "4.28.2"
   resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.28.2.tgz#7a8320f00141666813d0ae43b49ee8244f7cf92a"
@@ -6094,6 +6109,13 @@ libnpmpublish@^4.0.0:
     semver "^7.1.3"
     ssri "^8.0.1"
 
+libnpmsearch@^5.0.3:
+  version "5.0.3"
+  resolved "https://registry.yarnpkg.com/libnpmsearch/-/libnpmsearch-5.0.3.tgz#ed502a4c2c70ea36723180455fae1357546b2184"
+  integrity sha512-Ofq76qKAPhxbiyzPf/5LPjJln26VTKwU9hIU0ACxQ6tNtBJ1CHmI7iITrdp7vNezhZc0FlkXwrIpqXjhBJZgLQ==
+  dependencies:
+    npm-registry-fetch "^13.0.0"
+
 lines-and-columns@^1.1.6:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.1.6.tgz#1c00c743b433cd0a4e80758f7b64a57440d9ff00"
@@ -6986,7 +7008,7 @@ npm-registry-fetch@^11.0.0:
     minizlib "^2.0.0"
     npm-package-arg "^8.0.0"
 
-npm-registry-fetch@^13.0.1:
+npm-registry-fetch@^13.0.0, npm-registry-fetch@^13.0.1:
   version "13.1.1"
   resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-13.1.1.tgz#26dc4b26d0a545886e807748032ba2aefaaae96b"
   integrity sha512-5p8rwe6wQPLJ8dMqeTnA57Dp9Ox6GH9H60xkyJup07FmVlu3Mk7pf/kIIpl9gaN5bM8NM+UUx3emUWvDNTt39w==
@@ -9594,6 +9616,11 @@ yargs-parser@^20.2.2, yargs-parser@^20.2.3:
   resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-20.2.9.tgz#2eb7dc3b0289718fc295f362753845c41a0c94ee"
   integrity sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==
 
+yargs-parser@^21.0.0:
+  version "21.0.1"
+  resolved "https://registry.yarnpkg.com/yargs-parser/-/yargs-parser-21.0.1.tgz#0267f286c877a4f0f728fceb6f8a3e4cb95c6e35"
+  integrity sha512-9BK1jFpLzJROCI5TzwZL/TU4gqjK5xiHV/RfWLOahrjAko/e4DJkRDZQXfvqAsiZzzYhgAzbgz6lg48jcm4GLg==
+
 [email protected], yargs@^15.1.0, yargs@^15.3.1:
   version "15.3.1"
   resolved "https://registry.yarnpkg.com/yargs/-/yargs-15.3.1.tgz#9505b472763963e54afe60148ad27a330818e98b"
@@ -9641,6 +9668,19 @@ yargs@^16.2.0:
     y18n "^5.0.5"
     yargs-parser "^20.2.2"
 
+yargs@^17.5.1:
+  version "17.5.1"
+  resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.5.1.tgz#e109900cab6fcb7fd44b1d8249166feb0b36e58e"
+  integrity sha512-t6YAJcxDkNX7NFYiVtKvWUz8l+PaKTLiL63mJYWR2GnHq2gjEWISzsLp9wg3aY36dY1j+gfIEL3pIF+XlJJfbA==
+  dependencies:
+    cliui "^7.0.2"
+    escalade "^3.1.1"
+    get-caller-file "^2.0.5"
+    require-directory "^2.1.1"
+    string-width "^4.2.3"
+    y18n "^5.0.5"
+    yargs-parser "^21.0.0"
+
 zero-fill@^2.2.3:
   version "2.2.4"
   resolved "https://registry.yarnpkg.com/zero-fill/-/zero-fill-2.2.4.tgz#b041320973dbcb03cd90193270ac8d4a3da05fc1"