Add normalize-yaml package to normalize YAML consistently

.circleci/config.yml

@@ -196,6 +196,7 @@ jobs:
           name: Run Lints
           command: |
             make lint_yarn_lockfile
+            make lint_yaml
             yarn run lint
             yarn run typecheck
             bundle exec rubocop

Gemfile

@@ -125,7 +125,6 @@ group :test do
   gem 'rack-test', '>= 1.1.0'
   gem 'rails-controller-testing', '>= 1.0.4'
   gem 'rspec-retry'
-  gem 'rubypants-unicode', '~> 0.2.5'
   gem 'shoulda-matchers', '~> 4.0', require: false
   gem 'timecop'
   gem 'webdrivers', '~> 4.0'

Gemfile.lock

@@ -585,7 +585,6 @@ GEM
       rexml
     ruby-statistics (2.1.2)
     ruby2_keywords (0.0.2)
-    rubypants-unicode (0.2.5)
     rubyzip (1.3.0)
     safe_target_blank (1.0.2)
       rails
@@ -796,7 +795,6 @@ DEPENDENCIES
   rubocop-rails (>= 2.5.2)
   ruby-progressbar
   ruby-saml
-  rubypants-unicode (~> 0.2.5)
   safe_target_blank (>= 1.0.2)
   saml_idp!
   sassc-rails (~> 2.1.2)

Makefile

@@ -37,6 +37,9 @@ lint_erb:
 lint_yarn_lockfile:
 	(! git diff --name-only | grep yarn.lock) || (echo "Error: Sync Yarn lockfile using 'yarn install'"; exit 1)
 
+lint_yaml: normalize_yaml
+	(! git diff --name-only | grep "^config/.*\.yml$$") || (echo "Error: Run 'make normalize_yaml' to normalize YAML"; exit 1)
+
 lintfix:
 	@echo "--- rubocop fix ---"
 	bundle exec rubocop -R -a
@@ -71,8 +74,7 @@ run-https: tmp/$(HOST)-$(PORT).key tmp/$(HOST)-$(PORT).crt
 .PHONY: setup all lint run test check brakeman
 
 normalize_yaml:
-	i18n-tasks normalize
-	find ./config/locales -type f | xargs ./scripts/normalize-yaml config/country_dialing_codes.yml
+	find ./config/locales -type f | xargs yarn normalize-yaml config/country_dialing_codes.yml
 
 optimize_svg:
 	# Without disabling minifyStyles, keyframes are removed (e.g. `app/assets/images/id-card.svg`).
@@ -86,6 +88,7 @@ lint_optimized_assets: optimize_assets
 
 update_country_dialing_codes:
 	bundle exec ./scripts/pinpoint-supported-countries > config/country_dialing_codes.yml
+	yarn normalize-yaml config/country_dialing_codes.yml
 
 check_asset_strings:
 	find ./app/javascript -name "*.js*" | xargs ./scripts/check-assets

README.md

@@ -147,7 +147,7 @@ We recommend using [Homebrew](https://brew.sh/), [rbenv](https://github.com/rben
 
 #### Translations
 
-  Login.gov translates the IdP into English, French and Spanish. To help us handle extra newlines and make sure we wrap lines consistently, we have a script called `./scripts/normalize-yaml` that helps format YAML consistently. After importing translations (or making changes to the *.yml files with strings, run this for the IdP app:
+  Login.gov translates the IdP into English, French and Spanish. To help us handle extra newlines and make sure we wrap lines consistently, we have a script that helps format YAML consistently. After importing translations (or making changes to the `*.yml` files with strings, run this for the IdP app:
 
   ```
   $ make normalize_yaml
@@ -201,4 +201,4 @@ It's likely that you'll be prompted with a screen with warnings about an unsafe
 
 There was an initial attempt to dockerize the IDP but it is currently deprecated, mostly non-functional, and not maintained. There is ongoing work to make the IDP more [12 Factor](https://12factor.net/) compliant which will eventually lead to better support for containerization.
 
-If you'd like to work with the previous implementation see the [Docker documentation](./docs/Docker.md) to install the IdP as a container. 
+If you'd like to work with the previous implementation see the [Docker documentation](./docs/Docker.md) to install the IdP as a container.

app/javascript/packages/normalize-yaml/cli.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+import { promises as fsPromises } from 'fs';
+import { join } from 'path';
+import prettier from 'prettier';
+import normalize from './index.js';
+
+const { readFile, writeFile } = fsPromises;
+
+/** @type {Record<string,any>=} */
+const prettierConfig = prettier.resolveConfig.sync(process.cwd());
+
+const files = process.argv.slice(2);
+Promise.all(
+  files.map(async (relativePath) => {
+    const absolutePath = join(process.cwd(), relativePath);
+    const content = await readFile(absolutePath, 'utf8');
+    await writeFile(absolutePath, normalize(content, prettierConfig));
+  }),
+);

app/javascript/packages/normalize-yaml/index.js

@@ -0,0 +1,17 @@
+import YAML from 'yaml';
+import prettier from 'prettier';
+import visitors from './visitors/index.js';
+
+/**
+ * @param {string} content Original content.
+ * @param {Record<string,any>=} prettierConfig Optional Prettier configuration object.
+ *
+ * @return {string} Normalized content.
+ */
+function normalize(content, prettierConfig) {
+  const document = YAML.parseDocument(content);
+  YAML.visit(document, visitors);
+  return prettier.format(document.toString(), { ...prettierConfig, parser: 'yaml' });
+}
+
+export default normalize;

app/javascript/packages/normalize-yaml/index.spec.js

@@ -0,0 +1,32 @@
+import normalize from './index.js';
+
+describe('normalize', () => {
+  it('applies smart quotes', () => {
+    const original = '---\na: \'<strong class="example...">Hello "world"...</strong>\'';
+    const expected = '---\na: \'<strong class="example...">Hello “world”…</strong>\'\n';
+
+    expect(normalize(original)).to.equal(expected);
+  });
+
+  it('retains comments', () => {
+    const original = '---\n# Comment\na: true';
+    const expected = '---\n# Comment\na: true\n';
+
+    expect(normalize(original)).to.equal(expected);
+  });
+
+  it('sorts keys', () => {
+    const original = '---\nmap:\n  b: false\n  a: true';
+    const expected = '---\nmap:\n  a: true\n  b: false\n';
+
+    expect(normalize(original)).to.equal(expected);
+  });
+
+  it('formats using prettier', () => {
+    const original = "---\nfoo:  'bar' ";
+    const expected = '---\nfoo: "bar"\n';
+    const prettierConfig = { singleQuote: false };
+
+    expect(normalize(original, prettierConfig)).to.equal(expected);
+  });
+});

app/javascript/packages/normalize-yaml/package.json

@@ -0,0 +1,14 @@
+{
+  "name": "@18f/identity-normalize-yaml",
+  "private": true,
+  "version": "1.0.0",
+  "type": "module",
+  "bin": {
+    "normalize-yaml": "./cli.js"
+  },
+  "dependencies": {
+    "prettier": "^2.2.1",
+    "smartquotes": "^2.3.2",
+    "yaml": "^2.0.0-5"
+  }
+}

app/javascript/packages/normalize-yaml/visitors/format-content.js

@@ -0,0 +1,32 @@
+import smartquotes from 'smartquotes';
+
+/**
+ * @param {string} html String potentially containing HTML.
+ * @param {(substring: string) => string} replacer Replacer function given content between non-tag
+ * content to replace.
+ *
+ * @return {string}
+ */
+export function replaceInHTMLContent(html, replacer) {
+  return html.replace(
+    /([^<]*)(<.*?>)/g,
+    (_match, text, tag) => (text ? replacer(text) : text) + tag,
+  );
+}
+
+/**
+ * Replaces any instance of three dot characters with single ellipsis characters.
+ *
+ * @param {string} string
+ *
+ * @return {string}
+ */
+export const ellipses = (string) => string.replace(/\.\.\./g, '…');
+
+export default /** @type {import('yaml').visitor} */ ({
+  Scalar(_key, node) {
+    if (typeof node.value === 'string') {
+      node.value = replaceInHTMLContent(node.value, (string) => ellipses(smartquotes(string)));
+    }
+  },
+});

app/javascript/packages/normalize-yaml/visitors/format-content.spec.js

@@ -0,0 +1,29 @@
+import { replaceInHTMLContent, ellipses } from './format-content.js';
+
+describe('replaceInHTMLContent', () => {
+  it('replaces in html content', () => {
+    const string = '<div>This is a failure</div>';
+    const result = replaceInHTMLContent(string, (match) => match.replace('failure', 'success'));
+    const expected = '<div>This is a success</div>';
+
+    expect(result).to.equal(expected);
+  });
+
+  it('does not replace in html tags', () => {
+    const string = '<div data-div-type="div">div<div>div</div>div</div>';
+    const result = replaceInHTMLContent(string, (match) => match.replace('div', ''));
+    const expected = '<div data-div-type="div"><div></div></div>';
+
+    expect(result).to.equal(expected);
+  });
+});
+
+describe('ellipses', () => {
+  it('replaces all instances of dots', () => {
+    const string = 'You must first... before you can...';
+    const result = ellipses(string);
+    const expected = 'You must first… before you can…';
+
+    expect(result).to.equal(expected);
+  });
+});

app/javascript/packages/normalize-yaml/visitors/index.js

@@ -0,0 +1,7 @@
+import formatContent from './format-content.js';
+import sortKeys from './sort-keys.js';
+
+export default /** @type {import('yaml').visitor} */ ({
+  ...formatContent,
+  ...sortKeys,
+});

app/javascript/packages/normalize-yaml/visitors/sort-keys.js

@@ -0,0 +1,12 @@
+export default /** @type {import('yaml').visitor} */ ({
+  Map(_key, node) {
+    node.items.sort(
+      /**
+       * @param {import('yaml').Pair<any>} a
+       * @param {import('yaml').Pair<any>} b
+       * @return {number}
+       */
+      (a, b) => a.key.toString().localeCompare(b.key.toString()),
+    );
+  },
+});

config/locales/account/en.yml

@@ -4,13 +4,12 @@ en:
     connected_apps:
       associated: Connected %{timestamp}
       description: With your login.gov account, you can securely connect to multiple
-        government accounts online. Below is a list of all the accounts you currently
-        have connected.
+        government accounts online. Below is a list of all the accounts you
+        currently have connected.
     email_language:
-      default: "%{language} (default)"
+      default: '%{language} (default)'
       edit_title: Edit email language preference
-      languages_list: "%{app_name} allows you to receive your email communication
-        in %{list}."
+      languages_list: '%{app_name} allows you to receive your email communication in %{list}.'
       name:
         en: English
         es: Spanish
@@ -19,10 +18,11 @@ en:
       sentence_connector: or
       updated: Your email language preference has been updated.
     forget_all_browsers:
-      longer_description: Once you choose to ‘forget all browsers,’ we’ll need additional
-        information to know that it’s actually you signing in to your account. We’ll
-        ask for a multi-factor authentication method (such as text/SMS code or a security
-        key) each time you want to access your account.
+      longer_description: Once you choose to ‘forget all browsers,’ we’ll need
+        additional information to know that it’s actually you signing in to your
+        account. We’ll ask for a multi-factor authentication method (such as
+        text/SMS code or a security key) each time you want to access your
+        account.
     index:
       address: Current address
       auth_app_add: Add app
@@ -33,7 +33,7 @@ en:
       backup_codes_exist: Generated
       backup_codes_no_exist: Not generated
       default: default
-      device: "%{browser} on %{os}"
+      device: '%{browser} on %{os}'
       dob: Date of birth
       email: Email address
       email_add: Add email
@@ -76,11 +76,10 @@ en:
         ial1: If you have added your PIV/CAC card to your account, you may use it
           instead of your email and password.
         ial2: If you have added your PIV/CAC card to your account, you may use it
-          instead of your email. You will need to provide your password to unlock
-          your profile.
+          instead of your email. You will need to provide your password to
+          unlock your profile.
     navigation:
-      access_services: Access your government benefits and services from your login.gov
-        account.
+      access_services: Access your government benefits and services from your login.gov account.
       add_authentication_apps: Add authentication apps
       add_email: Add email address
       add_federal_id: Add Federal Employee ID
@@ -102,10 +101,10 @@ en:
       footer: Authenticate to view your information.
     revoke_consent:
       link_title: Disconnect
-      longer_description_html: Your information will no longer be shared with %{service_provider}.
-        To access %{service_provider} in the future, you must give your consent to
-        share your information. You can give consent by going to the %{service_provider}
-        site and logging in.
+      longer_description_html: Your information will no longer be shared with
+        %{service_provider}. To access %{service_provider} in the future, you
+        must give your consent to share your information. You can give consent
+        by going to the %{service_provider} site and logging in.
     security:
       link: Learn more at the Help Center
       text: Your profile information is locked for your security.