Add dry runs to CI

.github/workflows/ci.yml

@@ -36,6 +36,21 @@ jobs:
       run: yarn lint
     - name: test
       run: yarn test
+  publish-packages-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/publish-packages.yml
+    with:
+      dry-run: true
+  publish-registry-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/publish-registry.yml
+    with:
+      dry-run: true
+  retag-dry-run:
+    needs: build_and_test
+    uses: ./.github/workflows/retag.yml
+    with:
+      dry-run: true
   publish_alpha:
     name: publish alpha release
     runs-on: ubuntu-latest

.github/workflows/publish-packages.yml

@@ -0,0 +1,41 @@
+name: Publish packages
+on:
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  publish-packages:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Parse declarations
+        run: yarn workspace @definitelytyped/publisher parse
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Calculate versions
+        run: yarn workspace @definitelytyped/publisher calculate-versions
+      - name: Generate packages
+        run: yarn workspace @definitelytyped/publisher generate
+      - name: Publish packages${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn workspace @definitelytyped/publisher publish-packages${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }}
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/

.github/workflows/publish-registry.yml

@@ -0,0 +1,36 @@
+name: Publish registry
+on:
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  publish-registry:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - name: Parse declarations
+        run: yarn workspace @definitelytyped/publisher parse
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Publish registry${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn workspace @definitelytyped/publisher publish-registry${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/

.github/workflows/retag.yml

@@ -0,0 +1,38 @@
+name: Update npm tags
+on:
+  schedule:
+    # https://crontab.guru/#0_0_*_*_0
+    - cron: 0 0 * * 0
+  workflow_call:
+    inputs:
+      dry-run:
+        type: boolean
+  workflow_dispatch:
+    inputs:
+      dry-run:
+        type: boolean
+jobs:
+  retag:
+    if: github.event_name != 'schedule' || github.repository == 'microsoft/DefinitelyTyped-tools'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - run: yarn install --frozen-lockfile
+      - run: yarn build
+      - uses: actions/cache@v3
+        with:
+          path: packages/utils/cache/
+          key: cache-${{ github.run_id }}
+          restore-keys: cache-
+      - name: Update npm tags${{ (inputs || github.event.inputs).dry-run && ' dry run' || '' }}
+        run: yarn retag${{ (inputs || github.event.inputs).dry-run && ' --dry' || '' }}
+        env:
+          NPM_TOKEN: ${{ secrets.NPM_RETAG_TOKEN }}
+      - if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.job }}
+          path: packages/definitions-parser/data/

package.json

@@ -11,7 +11,7 @@
     "format": "prettier --write 'packages/**/*.ts'",
     "test": "jest",
     "build": "tsc -b .",
-    "retag": "node packages/retag/dist/retag.js"
+    "retag": "node --require source-map-support/register packages/retag/"
   },
   "devDependencies": {
     "@types/jest": "^25.1.3",

packages/publisher/src/calculate-versions.ts

@@ -1,4 +1,5 @@
-import { defaultLocalOptions } from "./lib/common";
+import * as process from "process";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { ChangedPackages, ChangedPackagesJson, ChangedTypingJson, versionsFilename } from "./lib/versions";
 import { getDefinitelyTyped, AllPackages, NotNeededPackage, writeDataFile } from "@definitelytyped/definitions-parser";
 import {
@@ -18,7 +19,11 @@ import { cacheDirPath } from "./lib/settings";
 if (!module.parent) {
   const log = loggerWithErrors()[0];
   logUncaughtErrors(async () =>
-    calculateVersions(await getDefinitelyTyped(defaultLocalOptions, log), new UncachedNpmInfoClient(), log)
+    calculateVersions(
+      await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log),
+      new UncachedNpmInfoClient(),
+      log
+    )
   );
 }
 

packages/publisher/src/generate-packages.ts

@@ -2,9 +2,10 @@ import { makeTypesVersionsForPackageJson } from "@definitelytyped/header-parser"
 import assert = require("assert");
 import { emptyDir, mkdir, mkdirp, readFileSync } from "fs-extra";
 import path = require("path");
+import * as process from "process";
 import yargs = require("yargs");
 
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { outputDirPath, sourceBranch, cacheDirPath } from "./lib/settings";
 import {
   assertNever,
@@ -44,7 +45,7 @@ if (!module.parent) {
   const tgz = !!yargs.argv.tgz;
   logUncaughtErrors(async () => {
     const log = loggerWithErrors()[0];
-    const dt = await getDefinitelyTyped(defaultLocalOptions, log);
+    const dt = await getDefinitelyTyped(process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions, log);
     const allPackages = await AllPackages.read(dt);
     await generatePackages(dt, allPackages, await readChangedPackages(allPackages), tgz);
   });

packages/publisher/src/parse-definitions.ts

@@ -1,7 +1,8 @@
 import os from "os";
+import * as process from "process";
 import yargs from "yargs";
 import { logUncaughtErrors, loggerWithErrors, assertDefined, FS } from "@definitelytyped/utils";
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { getTypingInfo } from "@definitelytyped/definitions-parser/dist/lib/definition-parser";
 import {
   getDefinitelyTyped,
@@ -16,7 +17,7 @@ if (!module.parent) {
     nProcesses: { type: "number" },
   }).argv;
 
-  const options = defaultLocalOptions;
+  const options = process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions;
   logUncaughtErrors(async () => {
     const log = loggerWithErrors()[0];
     const dt = await getDefinitelyTyped(options, log);

packages/publisher/src/publish-packages.ts

@@ -1,7 +1,8 @@
+import * as process from "process";
 import applicationinsights = require("applicationinsights");
 import * as yargs from "yargs";
 
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { deprecateNotNeededPackage, publishNotNeededPackage, publishTypingsPackage } from "./lib/package-publisher";
 import { getDefinitelyTyped, AllPackages } from "@definitelytyped/definitions-parser";
 import {
@@ -23,7 +24,10 @@ if (!module.parent) {
   const dry = !!yargs.argv.dry;
   const deprecateName = yargs.argv.deprecate as string | undefined;
   logUncaughtErrors(async () => {
-    const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]);
+    const dt = await getDefinitelyTyped(
+      process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions,
+      loggerWithErrors()[0]
+    );
     if (deprecateName !== undefined) {
       // A '--deprecate' command is available in case types-publisher got stuck *while* trying to deprecate a package.
       // Normally this should not be needed.
@@ -59,7 +63,7 @@ export default async function publishPackages(
     log("=== Publishing packages ===");
   }
 
-  const client = await NpmPublishClient.create(await getSecret(Secret.NPM_TOKEN), undefined);
+  const client = await NpmPublishClient.create(dry ? "" : await getSecret(Secret.NPM_TOKEN), undefined);
 
   for (const cp of changedPackages.changedTypings) {
     log(`Publishing ${cp.pkg.desc}...`);

packages/publisher/src/publish-registry.ts

@@ -1,8 +1,9 @@
 import assert = require("assert");
+import * as process from "process";
 import { emptyDir } from "fs-extra";
 import * as yargs from "yargs";
 
-import { defaultLocalOptions } from "./lib/common";
+import { defaultLocalOptions, defaultRemoteOptions } from "./lib/common";
 import { outputDirPath, validateOutputPath, cacheDirPath } from "./lib/settings";
 import {
   getDefinitelyTyped,
@@ -47,7 +48,10 @@ Generated by [types-publisher](${pkg.homepage}).`;
 if (!module.parent) {
   const dry = !!yargs.argv.dry;
   logUncaughtErrors(async () => {
-    const dt = await getDefinitelyTyped(defaultLocalOptions, loggerWithErrors()[0]);
+    const dt = await getDefinitelyTyped(
+      process.env.GITHUB_ACTIONS ? defaultRemoteOptions : defaultLocalOptions,
+      loggerWithErrors()[0]
+    );
     await publishRegistry(dt, await AllPackages.read(dt), dry, new UncachedNpmInfoClient());
   });
 }
@@ -86,7 +90,7 @@ export default async function publishRegistry(
     const packageJson = generatePackageJson(typesRegistry, newVersion, newContentHash);
     await generate(registry, packageJson);
 
-    const token = await getSecret(Secret.NPM_TOKEN);
+    const token = dry ? "" : await getSecret(Secret.NPM_TOKEN);
 
     const publishClient = () => NpmPublishClient.create(token, { defaultTag: "next" });
     if (!semver.eq(highestSemverVersion, npmVersion)) {

packages/retag/package.json

@@ -29,9 +29,6 @@
     "@types/yargs": "^15.0.5"
   },
   "main": "dist/index.js",
-  "bin": {
-    "retag": "./dist/retag.js"
-  },
   "publishConfig": {
     "access": "public"
   }

packages/retag/src/index.ts

@@ -52,10 +52,14 @@ async function main() {
  */
 async function tag(dry: boolean, nProcesses: number, name?: string) {
   const log = loggerWithErrors()[0];
-  const options = { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true };
+  const options = process.env.GITHUB_ACTIONS
+    ? { definitelyTypedPath: undefined, progress: false, parseInParallel: false }
+    : { definitelyTypedPath: "../DefinitelyTyped", progress: true, parseInParallel: true };
   await parseDefinitions(
     await getDefinitelyTyped(options, log),
-    { nProcesses: nProcesses || os.cpus().length, definitelyTypedPath: "../DefinitelyTyped" },
+    options.parseInParallel
+      ? { nProcesses: nProcesses || os.cpus().length, definitelyTypedPath: "../DefinitelyTyped" }
+      : undefined,
     log
   );