Refactored old server errors

src/main.ts

@@ -310,6 +310,8 @@ async function getTsServerRepoResult(
                         ? prettyPrintServerHarnessOutput(oldSpawnResult.stdout, /*filter*/ false)
                         : `Timed out after ${executionTimeout} ms`));
 
+            await fs.promises.writeFile(rawErrorPath, oldSpawnResult?.stdout ?? `Timed out after ${executionTimeout} ms`);
+
             // We don't want to drown PRs with comments.
             // Override the results to say nothing interesting changed.
             if (isPr && newServerFailed && oldSpawnResult) {
@@ -335,10 +337,7 @@ async function getTsServerRepoResult(
             installCommands,
         };
 
-        if (oldServerFailed && !newServerFailed) {
-            return { status: "Detected interesting changes", tsServerResult }
-        }
-        if (!newServerFailed) {
+        if (!oldServerFailed && !newServerFailed) {
             return { status: "Detected no interesting changes" };
         }
 
@@ -404,25 +403,77 @@ function createOldErrorSummary(summaries: Summary[]): string {
 
     let text = `
 <details>
-<summary>${errorMessage}</summary>
+<summary>New server no longer reports this error: ${errorMessage}</summary>
 
 \`\`\`
 ${oldServerError}
 \`\`\`
 
-<h4>Repos no longer reporting the error</h4>
-<ul>
-`;
+<h4>Affected repos</h4>`;
 
     for (const summary of summaries) {
         const owner = summary.repo.owner ? `${mdEscape(summary.repo.owner)}/` : "";
         const url = summary.repo.url ?? "";
 
-        text += `<li><a href="${url}">${owner + mdEscape(summary.repo.name)}</a></li>\n`
+        text += `
+<details>
+<summary><a href="${url}">${owner + mdEscape(summary.repo.name)}</a></summary>
+Raw error text: <code>${summary.rawErrorArtifactPath}</code> in the <a href="${artifactFolderUrlPlaceholder}">artifact folder</a> <br />
+Replay commands: <code>${summary.replayScriptArtifactPath}</code> in the <a href="${artifactFolderUrlPlaceholder}">artifact folder</a>
+<h4>Last few requests</h4>
+
+\`\`\`json
+${summary.replayScript}
+\`\`\`
+
+<h4>Repro steps</h4>
+
+\`\`\`bash
+#!/bin/bash
+
+`;
+        // No url means is user test repo
+        if (!summary.repo.url) {
+            text += `# Manually download user test \`${summary.repo.name}\`\n`;
+        }
+        else {
+            text += `git clone ${summary.repo.url} --recurse-submodules\n`;
+
+            if (summary.commit) {
+                text += `git -C "./${summary.repo.name}" reset --hard ${summary.commit}\n`;
+            }
+        }
+
+        if (summary.tsServerResult.installCommands.length > 1) {
+            text += "# Install packages (exact steps are below, but it might be easier to follow the repo readme)\n";
+        }
+        for (const command of summary.tsServerResult.installCommands) {
+            const workingDirFlag = command.tool === ip.InstallTool.Npm
+                ? "--prefix"
+                : command.tool === ip.InstallTool.Yarn
+                    ? "--cwd"
+                    : "--dir"; // pnpm
+
+            text += `${command.tool} ${workingDirFlag} "./${command.prettyDirectory}" ${command.arguments.join(" ")}\n`;
+        }
+
+        text += `downloadUrl=$(curl -s "${getArtifactsApiUrlPlaceholder}?artifactName=${summary.resultDirName}&api-version=7.0" | jq -r ".resource.downloadUrl")
+wget -O ${summary.resultDirName}.zip "$downloadUrl"
+unzip -p ${summary.resultDirName}.zip ${summary.resultDirName}/${summary.replayScriptName} > ${summary.replayScriptName}
+npm install --no-save @typescript/server-replay
+\`\`\`
+
+To run the repro:
+\`\`\`bash
+# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
+npx tsreplay ./${summary.repo.name} ./${summary.replayScriptName} <PATH_TO_tsserver.js>
+\`\`\`
+
+</details>
+`;
     }
 
     text += `
-</ul>
 </details>
 `;
 

src/postGithubIssue.ts

@@ -44,12 +44,11 @@ for (const path of metadataFilePaths) {
     }
 }
 
-const title = entrypoint === "tsserver"
-    ? `[ServerErrors][${language}] ${newTscResolvedVersion}`
-    : `[NewErrors] ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`;
+
+const title = `${entrypoint === "tsserver" ? `[ServerErrors][${language}]` : `[NewErrors]`} ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`;
 
 const description = entrypoint === "tsserver"
-    ? `The following errors were reported by ${newTscResolvedVersion}`
+    ? `The following errors were reported by ${newTscResolvedVersion} vs ${oldTscResolvedVersion}`
     : `The following errors were reported by ${newTscResolvedVersion}, but not by ${oldTscResolvedVersion}`;
 let header = `${description}
 [Pipeline that generated this bug](https://typescript.visualstudio.com/TypeScript/_build?definitionId=48)

test/__snapshots__/main.test.ts.snap

@@ -1,5 +1,191 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`main outputs old server errors 1`] = `
+[MockFunction] {
+  "calls": [
+    [
+      "<BASE_PATH>/ts_downloads/base/MockRepoOwner.MockRepoName.rawError.txt",
+      "Req #123 - cursedCommand
+Some error. Could not do something. 
+Maybe a Debug fail.
+    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
+    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
+    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
+    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
+    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)",
+    ],
+    [
+      "<BASE_PATH>/ts_downloads/base/MockRepoOwner.MockRepoName.rawError.txt",
+      "{"request_seq":"123","command":"cursedCommand","message":"Some error. Could not do something. \\nMaybe a Debug fail.\\n    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)\\n    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)\\n    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)\\n    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)\\n    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)"}",
+    ],
+    [
+      "<BASE_PATH>/RepoResults123/718e48b799650d4b66e5d80ad6bac7b9.results.txt",
+      "<h2>Maybe a Debug fail.</h2>
+
+\`\`\`
+Req #123 - cursedCommand
+    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
+    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
+    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
+    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
+    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)
+\`\`\`
+
+<h4>Affected repos</h4>
+<details>
+<summary><a href="https://github.com/MockRepoOwner/MockRepoName">MockRepoOwner/MockRepoName</a></summary>
+Raw error text: <code>RepoResults123/MockRepoOwner.MockRepoName.rawError.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a> <br />
+Replay commands: <code>RepoResults123/MockRepoOwner.MockRepoName.replay.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a>
+<h4>Last few requests</h4>
+
+\`\`\`json
+{"rootDirPlaceholder":"@PROJECT_ROOT@","serverArgs":["--disableAutomaticTypingAcquisition"]}
+{"seq":1,"type":"request","command":"configure","arguments":{"preferences":{"disableLineTextInReferences":true,"includePackageJsonAutoImports":"auto","includeCompletionsForImportStatements":true,"includeCompletionsWithSnippetText":true,"includeAutomaticOptionalChainCompletions":true,"includeCompletionsWithInsertText":true,"includeCompletionsWithClassMemberSnippets":true,"allowIncompleteCompletions":true,"includeCompletionsForModuleExports":false},"watchOptions":{"excludeDirectories":["**/node_modules"]}}}
+{"seq":2,"type":"request","command":"updateOpen","arguments":{"changedFiles":[],"closedFiles":[],"openFiles":[{"file":"@PROJECT_ROOT@/sample_repoName.config.js","projectRootPath":"@PROJECT_ROOT@"}]}}
+{"seq":3,"type":"request","command":"cursedCommand","arguments":{"file":"@PROJECT_ROOT@/src/sampleTsFile.ts","line":1,"offset":1,"includeExternalModuleExports":false,"triggerKind":1}}
+\`\`\`
+
+<h4>Repro steps</h4>
+
+\`\`\`bash
+#!/bin/bash
+
+git clone https://github.com/MockRepoOwner/MockRepoName --recurse-submodules
+git -C "./MockRepoName" reset --hard 57b462387e88aa7e363af0daf867a5dc1e83a935
+# Install packages (exact steps are below, but it might be easier to follow the repo readme)
+npm --prefix "./dirA" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+npm --prefix "./dirB/dirC" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+npm --prefix "./dirD/dirE/dirF" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+downloadUrl=$(curl -s "PLACEHOLDER_GETARTIFACTS_API?artifactName=RepoResults123&api-version=7.0" | jq -r ".resource.downloadUrl")
+wget -O RepoResults123.zip "$downloadUrl"
+unzip -p RepoResults123.zip RepoResults123/MockRepoOwner.MockRepoName.replay.txt > MockRepoOwner.MockRepoName.replay.txt
+npm install --no-save @typescript/server-replay
+\`\`\`
+
+To run the repro:
+\`\`\`bash
+# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
+npx tsreplay ./MockRepoName ./MockRepoOwner.MockRepoName.replay.txt <PATH_TO_tsserver.js>
+\`\`\`
+
+</details>
+",
+      {
+        "encoding": "utf-8",
+      },
+    ],
+    [
+      "<BASE_PATH>/RepoResults123/metadata.json",
+      "{"newTsResolvedVersion":"1.1.1","oldTsResolvedVersion":"0.0.0","statusCounts":{"Detected interesting changes":1}}",
+      {
+        "encoding": "utf-8",
+      },
+    ],
+    [
+      "<BASE_PATH>/ts_downloads/base/MockRepoOwner.MockRepoName.rawError.txt",
+      "{"request_seq":"123","command":"cursedCommand","message":"Some error. Could not do something. \\nMaybe a Debug fail.\\n    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)\\n    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)\\n    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)\\n    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)\\n    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)"}",
+    ],
+    [
+      "<BASE_PATH>/RepoResults123/!718e48b799650d4b66e5d80ad6bac7b9.results.txt",
+      "
+<details>
+<summary>New server no longer reports this error: Maybe a Debug fail.</summary>
+
+\`\`\`
+Req #123 - cursedCommand
+    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)
+    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)
+    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)
+    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
+    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)
+\`\`\`
+
+<h4>Affected repos</h4>
+<details>
+<summary><a href="https://github.com/MockRepoOwner/MockRepoName">MockRepoOwner/MockRepoName</a></summary>
+Raw error text: <code>RepoResults123/MockRepoOwner.MockRepoName.rawError.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a> <br />
+Replay commands: <code>RepoResults123/MockRepoOwner.MockRepoName.replay.txt</code> in the <a href="PLACEHOLDER_ARTIFACT_FOLDER">artifact folder</a>
+<h4>Last few requests</h4>
+
+\`\`\`json
+{"rootDirPlaceholder":"@PROJECT_ROOT@","serverArgs":["--disableAutomaticTypingAcquisition"]}
+{"seq":1,"type":"request","command":"configure","arguments":{"preferences":{"disableLineTextInReferences":true,"includePackageJsonAutoImports":"auto","includeCompletionsForImportStatements":true,"includeCompletionsWithSnippetText":true,"includeAutomaticOptionalChainCompletions":true,"includeCompletionsWithInsertText":true,"includeCompletionsWithClassMemberSnippets":true,"allowIncompleteCompletions":true,"includeCompletionsForModuleExports":false},"watchOptions":{"excludeDirectories":["**/node_modules"]}}}
+{"seq":2,"type":"request","command":"updateOpen","arguments":{"changedFiles":[],"closedFiles":[],"openFiles":[{"file":"@PROJECT_ROOT@/sample_repoName.config.js","projectRootPath":"@PROJECT_ROOT@"}]}}
+{"seq":3,"type":"request","command":"cursedCommand","arguments":{"file":"@PROJECT_ROOT@/src/sampleTsFile.ts","line":1,"offset":1,"includeExternalModuleExports":false,"triggerKind":1}}
+\`\`\`
+
+<h4>Repro steps</h4>
+
+\`\`\`bash
+#!/bin/bash
+
+git clone https://github.com/MockRepoOwner/MockRepoName --recurse-submodules
+git -C "./MockRepoName" reset --hard 57b462387e88aa7e363af0daf867a5dc1e83a935
+# Install packages (exact steps are below, but it might be easier to follow the repo readme)
+npm --prefix "./dirA" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+npm --prefix "./dirB/dirC" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+npm --prefix "./dirD/dirE/dirF" install --prefer-offline --no-audit --no-progress --legacy-peer-deps --ignore-scripts -q
+downloadUrl=$(curl -s "PLACEHOLDER_GETARTIFACTS_API?artifactName=RepoResults123&api-version=7.0" | jq -r ".resource.downloadUrl")
+wget -O RepoResults123.zip "$downloadUrl"
+unzip -p RepoResults123.zip RepoResults123/MockRepoOwner.MockRepoName.replay.txt > MockRepoOwner.MockRepoName.replay.txt
+npm install --no-save @typescript/server-replay
+\`\`\`
+
+To run the repro:
+\`\`\`bash
+# \`npx tsreplay --help\` to learn about helpful switches for debugging, logging, etc.
+npx tsreplay ./MockRepoName ./MockRepoOwner.MockRepoName.replay.txt <PATH_TO_tsserver.js>
+\`\`\`
+
+</details>
+
+</details>
+",
+      {
+        "encoding": "utf-8",
+      },
+    ],
+    [
+      "<BASE_PATH>/RepoResults123/metadata.json",
+      "{"newTsResolvedVersion":"1.1.1","oldTsResolvedVersion":"0.0.0","statusCounts":{"Detected interesting changes":1}}",
+      {
+        "encoding": "utf-8",
+      },
+    ],
+  ],
+  "results": [
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+    {
+      "type": "return",
+      "value": undefined,
+    },
+  ],
+}
+`;
+
 exports[`main outputs server errors 1`] = `
 [MockFunction] {
   "calls": [
@@ -14,6 +200,10 @@ Maybe a Debug fail.
     at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)
     at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)",
     ],
+    [
+      "<BASE_PATH>/ts_downloads/base/MockRepoOwner.MockRepoName.rawError.txt",
+      "{"request_seq":"123","command":"cursedCommand","message":"Some error. Could not do something. \\nMaybe a Debug fail.\\n    at a (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:1:1)\\n    at b (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:2:2)\\n    at c (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:3:3)\\n    at d (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:4:4)\\n    at e (/mnt/vss/_work/1/s/typescript-1.1.1/lib/typescript.js:5:5)"}",
+    ],
     [
       "<BASE_PATH>/RepoResults123/718e48b799650d4b66e5d80ad6bac7b9.results.txt",
       "<h2>Maybe a Debug fail.</h2>
@@ -91,6 +281,10 @@ npx tsreplay ./MockRepoName ./MockRepoOwner.MockRepoName.replay.txt <PATH_TO_tss
       "type": "return",
       "value": undefined,
     },
+    {
+      "type": "return",
+      "value": undefined,
+    },
   ],
 }
 `;