diff --git a/.github/workflows/validate-task.yml b/.github/workflows/validate-task.yml index 151cbfb..864ebcd 100644 --- a/.github/workflows/validate-task.yml +++ b/.github/workflows/validate-task.yml @@ -81,5 +81,49 @@ jobs: - name: Lint workflows step uses: raven-actions/actionlint@3d39aea434753780c3b3d4a1a31c854b4dbf49d7 # v2.2.0 - - name: Check line endings step - run: docker run --rm -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker@sha256:67b9e9b16a674e36f7c05919da789f03a01d343ca8423eb8797179399af07c00 # editorconfig-checker v3.4.0 + - name: Check EditorConfig step + run: docker run --rm -v "$PWD":/check --workdir /check mstruebing/editorconfig-checker:latest + + # Deterministic offline codegen drift guard. Regenerates the embedded data from the committed + # LanguageData/ with --skip-download (no network), formats with the same CSharpier the codegen workflow + # uses, then fails if the tree drifts from the committed *DataGen.cs / *.json. This catches an emitter or + # source-data change whose regenerated output was not committed in the same PR. It never fetches upstream, + # so registry drift can never trip it - refreshing from upstream is the scheduled codegen workflow's job. + codegen-drift: + name: Check codegen drift job + runs-on: ubuntu-latest + + steps: + + - name: Setup .NET SDK step + uses: actions/setup-dotnet@26b0ec14cb23fa6904739307f278c14f94c95bf1 # v5.4.0 + with: + dotnet-version: 10.x + + - name: Checkout code step + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Regenerate codegen offline step + run: | + set -euo pipefail + dotnet run --project ./LanguageTagsCreate/LanguageTagsCreate.csproj -- \ + --codepath . --skip-download + + - name: Format code step + run: | + set -euo pipefail + dotnet tool restore + dotnet csharpier format . + + - name: Check for drift step + run: | + set -euo pipefail + # Fail on any drift - modified tracked files or new untracked generated files. Report a porcelain + # summary plus a diffstat rather than the full patch, which for generated data files can be multi-MB + # and blow up the Actions log. + if [ -n "$(git status --porcelain --untracked-files=all)" ]; then + echo "::error::Generated code or formatting drifted - regenerate offline (--skip-download), run 'dotnet csharpier format .', and commit the result" + git status --porcelain --untracked-files=all + git diff --stat + exit 1 + fi diff --git a/.husky/pre-commit b/.husky/pre-commit index 818853f..e9a75d5 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,9 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" -dotnet husky run +# Local pre-commit: language formatting and style only (no Docker). Full lint runs in CI and the VS Code Lint tasks. + +# .NET: CSharpier + dotnet format style via Husky.Net. A Python repo runs ruff here instead. +if command -v dotnet >/dev/null 2>&1; then + dotnet husky run +fi diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 10e227c..0604256 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -142,6 +142,63 @@ "CSharpier Format" ], "problemMatcher": [] + }, + // Lint group - the local full doc-lint surface (Docker at :latest), matching the CI lint set. + // Run on demand. The pre-commit hook does language formatting only. Every repo carries these. + { + "label": "Lint: EditorConfig", + "type": "process", + "command": "docker", + "args": [ "run", "--rm", "--pull=always", "-v", "${workspaceFolder}:/check", "-w", "/check", "mstruebing/editorconfig-checker:latest" ], + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Workflows", + "type": "process", + "command": "docker", + "args": [ "run", "--rm", "--pull=always", "-v", "${workspaceFolder}:/repo", "-w", "/repo", "rhysd/actionlint:latest", "-color" ], + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Markdown", + "type": "process", + "command": "docker", + "args": [ "run", "--rm", "--pull=always", "-v", "${workspaceFolder}:/workdir", "-w", "/workdir", "davidanson/markdownlint-cli2:latest", "**/*.md" ], + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: Spelling", + "type": "process", + "command": "docker", + "args": [ "run", "--rm", "--pull=always", "-v", "${workspaceFolder}:/workdir", "-w", "/workdir", "ghcr.io/streetsidesoftware/cspell:latest", "--no-progress", "README.md", "HISTORY.md" ], + "problemMatcher": [], + "presentation": { + "showReuseMessage": false, + "clear": false + } + }, + { + "label": "Lint: All", + "dependsOrder": "sequence", + "dependsOn": [ + "Lint: EditorConfig", + "Lint: Workflows", + "Lint: Markdown", + "Lint: Spelling" + ], + "problemMatcher": [] } ] } diff --git a/AGENTS.md b/AGENTS.md index e9ef32b..c97088f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -156,8 +156,8 @@ Anti-pattern: don't keep flipping the code on the same style point. Flip the rul - **Config files.** [`.editorconfig`](./.editorconfig) (per-file-type EOL plus the C# / ReSharper style block), [`.gitattributes`](./.gitattributes), [`.markdownlint-cli2.jsonc`](./.markdownlint-cli2.jsonc), [`CODESTYLE.md`](./CODESTYLE.md) (C# code style), and [`.github/copilot-instructions.md`](./.github/copilot-instructions.md) (the Copilot review runbook) hold the repo's formatting, linting, and review-mechanics rules. `CODESTYLE.md` sits at the repo root because `AGENTS.md` and `copilot-instructions.md` link it by relative path. Keep `copilot-instructions.md` narrow (Copilot/VS Code mechanics plus the commit/PR-title summary); project-specific conventions and the public-API contract live in this file, not there. - **Clean-compile gate.** Husky.Net pre-commit git hooks run the C# clean-compile checks (CSharpier format, then `dotnet format style --verify-no-changes`), installed with `dotnet tool restore` + `dotnet husky install`. The [`.vscode/tasks.json`](./.vscode/tasks.json) tasks `.NET Build`, `CSharpier Format`, and `.NET Format` are the canonical task names (owned by the `CODESTYLE.md` ".NET" section); do not loosen them. CI is the authoritative backstop: the `lint` job ([`WORKFLOW.md`](./WORKFLOW.md) D1.3) enforces CSharpier, `dotnet format style`, `markdownlint`, scoped `cspell`, and `actionlint` from the same config files, because a local hook can be bypassed. Keep the editor task, the hook, and CI in sync (CODESTYLE "Clean-Compile Verification"). -- **Linting tools.** CI is the authoritative lint run; a local run is only for fast feedback. The `dotnet` checks need only the .NET SDK: `dotnet format style` is built into the SDK, and CSharpier is restored by `dotnet tool restore` against [`.config/dotnet-tools.json`](./.config/dotnet-tools.json). The markdown, spelling, and workflow linters have no committed manifest; run each from its official Docker image, the portable path that avoids a local Node or Go install, mounting the repo as the working directory: `cspell` from `ghcr.io/streetsidesoftware/cspell`, `markdownlint-cli2` from `davidanson/markdownlint-cli2`, and `actionlint` (which bundles `shellcheck`) from `rhysd/actionlint`, pinned to the versions [`validate-task.yml`](./.github/workflows/validate-task.yml) uses. Each takes the file globs directly, for example `docker run --rm -v "$PWD":/work -w /work ghcr.io/streetsidesoftware/cspell cspell README.md HISTORY.md` or `... davidanson/markdownlint-cli2 '**/*.md'`. The cspell accepted-word list and the path exclusions both live in [`cspell.json`](./cspell.json), the single source: the Code Spell Checker extension reads `cspell.json` ahead of the workspace `cSpell` settings (so GUI "Add to dictionary" lands words there), and the CLI and CI read the same file. Do not keep a parallel word list in the `.code-workspace` file. A local cspell or markdownlint result that reports zero files checked scanned nothing; ignore it. There is intentionally no wrapper script; the editor, these Docker images, and CI are the supported runners. -- **Codegen.** Embedded language data is regenerated by [`LanguageTagsCreate/`](./LanguageTagsCreate/), which pulls directly from the official ISO 639-2/3 + RFC 5646 registries. There is no external codegen API key. +- **Linting tools.** CI is the authoritative lint run; a local run is only for fast feedback. The `dotnet` checks need only the .NET SDK: `dotnet format style` is built into the SDK, and CSharpier is restored by `dotnet tool restore` against [`.config/dotnet-tools.json`](./.config/dotnet-tools.json). The markdown, spelling, and workflow linters have no committed manifest; run each from its official Docker image, the portable path that avoids a local Node or Go install, mounting the repo as the working directory: `cspell` from `ghcr.io/streetsidesoftware/cspell`, `markdownlint-cli2` from `davidanson/markdownlint-cli2`, and `actionlint` (which bundles `shellcheck`) from `rhysd/actionlint`, at `:latest`. CI runs these three as pinned action wrappers (Dependabot bumps them) and editorconfig-checker via Docker `:latest`; local Docker runs and the VS Code **Lint** tasks track `:latest`. Each takes the file globs directly, for example `docker run --rm -v "$PWD":/work -w /work ghcr.io/streetsidesoftware/cspell README.md HISTORY.md` or `... davidanson/markdownlint-cli2 '**/*.md'`. The cspell accepted-word list and the path exclusions both live in [`cspell.json`](./cspell.json), the single source: the Code Spell Checker extension reads `cspell.json` ahead of the workspace `cSpell` settings (so GUI "Add to dictionary" lands words there), and the CLI and CI read the same file. Do not keep a parallel word list in the `.code-workspace` file. A local cspell or markdownlint result that reports zero files checked scanned nothing; ignore it. There is intentionally no wrapper script; the editor, these Docker images, and CI are the supported runners. +- **Codegen.** Embedded language data is regenerated by [`LanguageTagsCreate/`](./LanguageTagsCreate/), which pulls directly from the official ISO 639-2/3 + RFC 5646 registries. There is no external codegen API key. If you change a codegen emitter (`SaveCodeAsync` / `SaveJsonAsync` / `GetCodeGenString` in `LanguageTags/*Data.cs` or `LanguageTags/LanguageSchema.cs`) or the committed `LanguageData/` source, regenerate offline and commit the regenerated `LanguageTags/*DataGen.cs` and `LanguageData/*.json` in the **same PR**: `dotnet run --project ./LanguageTagsCreate/LanguageTagsCreate.csproj -- --codepath . --skip-download` then `dotnet tool restore && dotnet csharpier format .` (CSharpier is a local dotnet tool, so restore it first on a fresh clone). The `--skip-download` flag regenerates from the committed `LanguageData/` with no network (a full refresh from upstream, the default, is the scheduled codegen workflow's job, not something to run in a feature PR). The `validate-task` `codegen-drift` job runs exactly this on every push and fails if the regenerated output drifts from what is committed. - **Release notes.** Keep a short summary in [`README.md`](./README.md) and the full history in [`HISTORY.md`](./HISTORY.md); update both when cutting a release. ## Workflow YAML Conventions diff --git a/LanguageTagsCreate/CommandLine.cs b/LanguageTagsCreate/CommandLine.cs index f2eee13..2bb0a3e 100644 --- a/LanguageTagsCreate/CommandLine.cs +++ b/LanguageTagsCreate/CommandLine.cs @@ -9,6 +9,7 @@ internal sealed class CommandLine private readonly Option _logFileOption = CreateLogFileOption(); private readonly Option _logFileClearOption = CreateLogFileClearOption(); private readonly Option _codePathOption = CreateCodePathOption(); + private readonly Option _skipDownloadOption = CreateSkipDownloadOption(); private static readonly FrozenSet s_cliBypassList = FrozenSet.Create( StringComparer.OrdinalIgnoreCase, @@ -33,6 +34,7 @@ internal RootCommand CreateRootCommand() _logFileOption, _logFileClearOption, _codePathOption, + _skipDownloadOption, }; rootCommand.SetAction( (parseResult, cancellationToken) => @@ -55,6 +57,7 @@ internal Options CreateOptions(ParseResult parseResult) => FileClear = parseResult.GetValue(_logFileClearOption), }, CodePath = parseResult.GetValue(_codePathOption)!, + SkipDownload = parseResult.GetValue(_skipDownloadOption), }; private static Option CreateLogFileClearOption() => @@ -82,6 +85,13 @@ private static Option CreateLogFileOption() return option.AcceptLegalFileNamesOnly(); } + private static Option CreateSkipDownloadOption() => + new("--skip-download", "-s") + { + Description = + "Regenerate from the committed LanguageData/ directory without downloading, offline (default: false).", + }; + private static Option CreateCodePathOption() { Option option = new("--codepath", "-p") @@ -103,5 +113,6 @@ internal sealed class Options { internal required LoggerFactory.Options LogOptions { get; init; } internal required DirectoryInfo CodePath { get; init; } + internal bool SkipDownload { get; init; } } } diff --git a/LanguageTagsCreate/CreateTagData.cs b/LanguageTagsCreate/CreateTagData.cs index 0adf845..db5b31d 100644 --- a/LanguageTagsCreate/CreateTagData.cs +++ b/LanguageTagsCreate/CreateTagData.cs @@ -27,29 +27,63 @@ internal async Task DownloadDataAsync() { // Download all the data files Log.Information("Downloading all language tag data files ..."); + ResolveDataFilePaths(); Log.Information("Downloading ISO 639-2 data ..."); - _iso6392DataFile = Path.Combine(dataDirectory, Iso6392Data.DataFileName); - await DownloadFileAsync(new Uri(Iso6392Data.DataUri), _iso6392DataFile) + await DownloadFileAsync(new Uri(Iso6392Data.DataUri), _iso6392DataFile!) .ConfigureAwait(false); Log.Information("Downloading ISO 639-3 data ..."); - _iso6393DataFile = Path.Combine(dataDirectory, Iso6393Data.DataFileName); - await DownloadFileAsync(new Uri(Iso6393Data.DataUri), _iso6393DataFile) + await DownloadFileAsync(new Uri(Iso6393Data.DataUri), _iso6393DataFile!) .ConfigureAwait(false); Log.Information("Downloading RFC 5646 data ..."); - _rfc5646DataFile = Path.Combine(dataDirectory, Rfc5646Data.DataFileName); - await DownloadFileAsync(new Uri(Rfc5646Data.DataUri), _rfc5646DataFile) + await DownloadFileAsync(new Uri(Rfc5646Data.DataUri), _rfc5646DataFile!) .ConfigureAwait(false); Log.Information("Downloading UN M.49 data ..."); - _unM49DataFile = Path.Combine(dataDirectory, UnM49Data.DataFileName); - await DownloadFileAsync(new Uri(UnM49Data.DataUri), _unM49DataFile).ConfigureAwait(false); + await DownloadFileAsync(new Uri(UnM49Data.DataUri), _unM49DataFile!).ConfigureAwait(false); Log.Information("Language tag data files downloaded successfully."); } + internal void UseExistingData() + { + // Offline mode: regenerate from the already-committed LanguageData/ directory with no network + // access. Resolve the same data file paths a download would, then verify each is present. + Log.Information( + "Using existing language tag data files in {DataDirectory} ...", + dataDirectory + ); + ResolveDataFilePaths(); + + string[] dataFiles = + [ + _iso6392DataFile!, + _iso6393DataFile!, + _rfc5646DataFile!, + _unM49DataFile!, + ]; + foreach (string dataFile in dataFiles) + { + if (!File.Exists(dataFile)) + { + throw new FileNotFoundException( + $"Required data file does not exist: {dataFile}", + dataFile + ); + } + } + } + + private void ResolveDataFilePaths() + { + _iso6392DataFile = Path.Combine(dataDirectory, Iso6392Data.DataFileName); + _iso6393DataFile = Path.Combine(dataDirectory, Iso6393Data.DataFileName); + _rfc5646DataFile = Path.Combine(dataDirectory, Rfc5646Data.DataFileName); + _unM49DataFile = Path.Combine(dataDirectory, UnM49Data.DataFileName); + } + internal async Task CreateJsonDataAsync() { ArgumentNullException.ThrowIfNull(_iso6392DataFile); diff --git a/LanguageTagsCreate/Program.cs b/LanguageTagsCreate/Program.cs index 8057d66..df50b2d 100644 --- a/LanguageTagsCreate/Program.cs +++ b/LanguageTagsCreate/Program.cs @@ -64,9 +64,17 @@ internal async Task ExecuteAsync() return 1; } - // Download data files + // Obtain the source data: download fresh from upstream, or, in offline mode, use the data + // already committed under LanguageData/ so codegen can be re-exercised with no network. CreateTagData createTagData = new(dataDirectory, codeDirectory, cancellationToken); - await createTagData.DownloadDataAsync().ConfigureAwait(false); + if (commandLineOptions.SkipDownload) + { + createTagData.UseExistingData(); + } + else + { + await createTagData.DownloadDataAsync().ConfigureAwait(false); + } // Convert data files to JSON await createTagData.CreateJsonDataAsync().ConfigureAwait(false);