feat(translations/differences): visualize how many commits behind translations are (#8338)

* PoC: visualization for how far behind the lastest commit

* feat: tag for Metadata does not exist.

* fix: filter available

* refactor: execSync cwd option

* fix: filter priority

* Optimization through cache

* reduce ja time to 214s with git rev-list

* feat: source commit error report

* Optimize source commit logic
Generate source commit error report

* feat: source commit importance color

* new approach for traversing git commit graph

Co-authored-by: LeoMcA <[email protected]>

* remove p tag in table tag

Co-authored-by: Leo McArdle <[email protected]>

* add details to report message and file name

Co-authored-by: LeoMcA <[email protected]>

* modify source commit filter condition

Co-authored-by: Leo McArdle <[email protected]>

* fix: consider sourceCommitsBehindCount undefined

* check automatic cache validation

* refactor: types and naming

* execSync -> spawn, various fixes for optimization

Co-authored-by: LeoMcA <[email protected]>

* fix: make sourceCommitCache work correctly

* remove log

---------

Co-authored-by: LeoMcA <[email protected]>
Co-authored-by: Leo McArdle <[email protected]>

Author: hochan222

Diff for: .gitignore

@@ -103,6 +103,8 @@ testing/content/files/en-us/markdown/tool/m2h/index.html
 # eslintcache
 client/.eslintcache
 popularities.json
+source-commit.json
+source-commit-invalid-report.txt
 
 client/src/.linaria_cache
 ssr/.linaria-cache/

Diff for: client/src/translations/differences/index.tsx

@@ -35,6 +35,8 @@ interface DocumentEdits {
   parentModified: string;
   commitURL: string;
   parentCommitURL: string;
+  sourceCommitsBehindCount?: number;
+  sourceCommitURL?: string;
 }
 
 interface Document {
@@ -574,6 +576,10 @@ function DocumentsTable({
         const a = A.mdn_url;
         const b = B.mdn_url;
         return reverse * a.localeCompare(b);
+      } else if (sort === "sourceCommit") {
+        const a = A.edits.sourceCommitsBehindCount ?? -1;
+        const b = B.edits.sourceCommitsBehindCount ?? -1;
+        return reverse * (b - a);
       } else {
         throw new Error(`Unrecognized sort '${sort}'`);
       }
@@ -606,6 +612,7 @@ function DocumentsTable({
             <TableHead id="popularity" title="Popularity" />
             <TableHead id="modified" title="Last modified" />
             <TableHead id="differences" title="Differences" />
+            <TableHead id="sourceCommit" title="Source Commit" />
           </tr>
         </thead>
         <tbody>
@@ -658,6 +665,14 @@ function DocumentsTable({
                     <LastModified edits={doc.edits} />
                   </td>
                   <td>{doc.differences.total.toLocaleString()}</td>
+                  <td>
+                    <L10nSourceCommitModified
+                      sourceCommitsBehindCount={
+                        doc.edits.sourceCommitsBehindCount
+                      }
+                      sourceCommitURL={doc.edits.sourceCommitURL}
+                    />
+                  </td>
                 </tr>
               );
             })}
@@ -680,6 +695,29 @@ function DocumentsTable({
   );
 }
 
+function L10nSourceCommitModified({
+  sourceCommitsBehindCount,
+  sourceCommitURL,
+}: Pick<DocumentEdits, "sourceCommitsBehindCount" | "sourceCommitURL">) {
+  if (
+    !sourceCommitURL ||
+    (!sourceCommitsBehindCount && sourceCommitsBehindCount !== 0)
+  ) {
+    return <>Metadata does not exist.</>;
+  }
+
+  const getImportanceColor = () => {
+    if (sourceCommitsBehindCount === 0) return "🟢";
+    return sourceCommitsBehindCount < 10 ? "🟠" : "🔴";
+  };
+
+  return (
+    <a
+      href={sourceCommitURL}
+    >{`${getImportanceColor()} ${sourceCommitsBehindCount} commits behind`}</a>
+  );
+}
+
 function LastModified({ edits }: { edits: DocumentEdits }) {
   const modified = dayjs(edits.modified);
   const parentModified = dayjs(edits.parentModified);

Diff for: server/translations.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 
 import express from "express";
 import { fdir } from "fdir";
+import { execSync, spawn } from "node:child_process";
 
 import { getPopularities, Document, Translation } from "../content/index.js";
 import {
@@ -59,8 +60,52 @@ function packageTranslationDifferences(translationDifferences) {
   return { total, countByType };
 }
 
+type RecentRepoHashType = string;
+
 const _foundDocumentsCache = new Map();
+const sourceCommitCache = fs.existsSync("./source-commit.json")
+  ? new Map<string, number | RecentRepoHashType>(
+      Object.entries(
+        JSON.parse(fs.readFileSync("./source-commit.json", "utf8"))
+      )
+    )
+  : new Map<string, number>();
+const commitFiles = new Map<string, string[]>();
+let commitFilesOldest = "HEAD";
 export async function findDocuments({ locale }) {
+  function checkCacheValidation(prevCache: Map<any, any>): void {
+    const contentHash = getRecentRepoHash(CONTENT_ROOT);
+    const translatedContentHash = getRecentRepoHash(CONTENT_TRANSLATED_ROOT);
+
+    function getRecentRepoHash(cwd: string): string {
+      return execSync("git rev-parse HEAD", { cwd }).toString().trimEnd();
+    }
+    function updateRecentRepoHash(cache: Map<string, any>): void {
+      cache.set(CONTENT_ROOT, contentHash);
+      cache.set(CONTENT_TRANSLATED_ROOT, translatedContentHash);
+    }
+    function isValidCache(cache: Map<string, any>): boolean {
+      return (
+        cache.has(CONTENT_ROOT) &&
+        cache.has(CONTENT_TRANSLATED_ROOT) &&
+        cache.get(CONTENT_ROOT) === contentHash &&
+        cache.get(CONTENT_TRANSLATED_ROOT) === translatedContentHash
+      );
+    }
+
+    if (isValidCache(sourceCommitCache)) {
+      return;
+    }
+    if (!isValidCache(prevCache)) {
+      prevCache.clear();
+      sourceCommitCache.clear();
+      commitFiles.clear();
+      commitFilesOldest = "HEAD";
+      updateRecentRepoHash(prevCache);
+      updateRecentRepoHash(sourceCommitCache);
+    }
+  }
+
   const counts = {
     // Number of documents found that aren't skipped
     found: 0,
@@ -81,6 +126,7 @@ export async function findDocuments({ locale }) {
   });
   counts.total = documentsFound.count;
 
+  checkCacheValidation(_foundDocumentsCache);
   if (!_foundDocumentsCache.has(locale)) {
     _foundDocumentsCache.set(locale, new Map());
   }
@@ -91,7 +137,7 @@ export async function findDocuments({ locale }) {
 
     if (!cache.has(filePath) || cache.get(filePath).mtime < mtime) {
       counts.cacheMisses++;
-      const document = getDocument(filePath);
+      const document = await getDocument(filePath);
       cache.set(filePath, {
         document,
         mtime,
@@ -114,14 +160,20 @@ export async function findDocuments({ locale }) {
     took,
   };
 
+  fs.writeFileSync(
+    "./source-commit.json",
+    JSON.stringify(Object.fromEntries(sourceCommitCache)),
+    "utf8"
+  );
+
   return {
     counts,
     times,
     documents,
   };
 }
 
-function getDocument(filePath) {
+async function getDocument(filePath) {
   function packagePopularity(document, parentDocument) {
     return {
       value: document.metadata.popularity,
@@ -141,34 +193,162 @@ function getDocument(filePath) {
     };
   }
 
-  function packageEdits(document, parentDocument) {
-    const commitURL = getLastCommitURL(
-      document.fileInfo.root,
-      document.metadata.hash
-    );
-    const parentCommitURL = getLastCommitURL(
-      parentDocument.fileInfo.root,
-      parentDocument.metadata.hash
-    );
-    const modified = document.metadata.modified;
-    const parentModified = parentDocument.metadata.modified;
+  function recordInvalidSourceCommit(
+    fileFolder: string,
+    commitHash: string,
+    message: string
+  ) {
+    const filePath = "./source-commit-invalid-report.txt";
+    const errorMessage = `- ${commitHash} commit hash is invalid in ${fileFolder}: ${message.replace(
+      /\n/g,
+      " "
+    )}`;
+    if (!fs.existsSync(filePath)) {
+      fs.writeFileSync(filePath, "");
+    }
+
+    fs.appendFile(filePath, `${errorMessage}\n`, function (err) {
+      if (err) throw err;
+    });
+  }
+
+  class GitError extends Error {
+    constructor(stderr: string) {
+      super(stderr);
+      this.name = "GitError";
+    }
+  }
+
+  function fillMemStore(commitHash: string) {
+    return new Promise((resolve, reject) => {
+      const git = spawn(
+        "git",
+        [
+          "log",
+          "--pretty=format:%x00%x00%H",
+          "--name-only",
+          "-z",
+          `${commitHash}..${commitFilesOldest}`,
+        ],
+        {
+          cwd: CONTENT_ROOT,
+        }
+      );
+
+      let stdoutBuffer = "";
+
+      git.stdout.on("data", (data) => {
+        stdoutBuffer += data.toString();
+        const commits = stdoutBuffer.split("\0\0");
+        const partial = commits.pop();
+        stdoutBuffer = partial;
+        commits.forEach((commit) => {
+          const [dirtyHash, files] = commit.split("\n");
+          // necessary for commits following those with no changes:
+          const hash = dirtyHash.replace(/\0/g, "");
+          commitFiles.set(hash, files ? files.split("\0") : []);
+        });
+      });
+
+      let stderr = "";
+
+      git.stderr.on("data", (data) => {
+        stderr += data.toString();
+      });
+
+      git.on("close", (code) => {
+        commitFilesOldest = commitHash;
+        code ? reject(new GitError(stderr)) : resolve(null);
+      });
+    });
+  }
+
+  async function getCommitBehindFromLatest(
+    fileFolder: string,
+    parentFilePath: string,
+    commitHash: string
+  ): Promise<number> {
+    if (sourceCommitCache.has(fileFolder)) {
+      return sourceCommitCache.get(fileFolder) as number;
+    }
+
+    try {
+      let count = 0;
+      if (!commitFiles.has(commitHash)) {
+        await fillMemStore(commitHash);
+      }
+      for (const [hash, files] of commitFiles.entries()) {
+        if (hash === commitHash) {
+          if (!files.includes(parentFilePath)) {
+            recordInvalidSourceCommit(
+              fileFolder,
+              commitHash,
+              "file isn't changed in this commit"
+            );
+          }
+          break;
+        }
+        if (files.includes(parentFilePath)) count++;
+      }
+      sourceCommitCache.set(fileFolder, count);
+    } catch (err) {
+      if (err instanceof GitError) {
+        recordInvalidSourceCommit(fileFolder, commitHash, err.message);
+      } else {
+        throw err;
+      }
+    }
+
+    return sourceCommitCache.get(fileFolder) as number;
+  }
+
+  async function packageEdits(document, parentDocument) {
+    const {
+      fileInfo: { root: fileRoot, folder: fileFolder },
+      metadata: { hash: fileHash, modified, l10n },
+    } = document;
+    const {
+      fileInfo: { root: parentFileRoot, path: parentFilePath },
+      metadata: { hash: parentFileHash, parentModified },
+    } = parentDocument;
+
+    const commitURL = getLastCommitURL(fileRoot, fileHash);
+    const parentCommitURL = getLastCommitURL(parentFileRoot, parentFileHash);
+    let sourceCommitURL;
+    let sourceCommitsBehindCount;
+
+    if (l10n?.sourceCommit) {
+      sourceCommitURL = getLastCommitURL(CONTENT_ROOT, l10n.sourceCommit);
+      sourceCommitsBehindCount = await getCommitBehindFromLatest(
+        fileFolder,
+        parentFilePath.replace(parentFileRoot, "files"),
+        l10n.sourceCommit
+      );
+    }
+
     return {
       commitURL,
       parentCommitURL,
       modified,
       parentModified,
+      sourceCommitURL,
+      sourceCommitsBehindCount,
     };
   }
 
   // We can't just open the `index.json` and return it like that in the XHR
   // payload. It's too much stuff and some values need to be repackaged/
   // serialized or some other transformation computation.
-  function packageDocument(document, englishDocument, translationDifferences) {
+  async function packageDocument(
+    document,
+    englishDocument,
+    translationDifferences
+  ) {
     const mdn_url = document.url;
     const { title } = document.metadata;
     const popularity = packagePopularity(document, englishDocument);
     const differences = packageTranslationDifferences(translationDifferences);
-    const edits = packageEdits(document, englishDocument);
+    const edits = await packageEdits(document, englishDocument);
     return { popularity, differences, edits, mdn_url, title };
   }
 
@@ -191,7 +371,7 @@ function getDocument(filePath) {
   )) {
     differences.push(difference);
   }
-  return packageDocument(document, englishDocument, differences);
+  return await packageDocument(document, englishDocument, differences);
 }
 
 const _defaultLocaleDocumentsCache = new Map();