Core: Enhance environment variable expansion in configuration

.yarn/versions/76751ff4.yml

@@ -0,0 +1,36 @@
+releases:
+  "@yarnpkg/cli": minor
+  "@yarnpkg/core": minor
+
+declined:
+  - "@yarnpkg/plugin-catalog"
+  - "@yarnpkg/plugin-compat"
+  - "@yarnpkg/plugin-constraints"
+  - "@yarnpkg/plugin-dlx"
+  - "@yarnpkg/plugin-essentials"
+  - "@yarnpkg/plugin-exec"
+  - "@yarnpkg/plugin-file"
+  - "@yarnpkg/plugin-git"
+  - "@yarnpkg/plugin-github"
+  - "@yarnpkg/plugin-http"
+  - "@yarnpkg/plugin-init"
+  - "@yarnpkg/plugin-interactive-tools"
+  - "@yarnpkg/plugin-jsr"
+  - "@yarnpkg/plugin-link"
+  - "@yarnpkg/plugin-nm"
+  - "@yarnpkg/plugin-npm"
+  - "@yarnpkg/plugin-npm-cli"
+  - "@yarnpkg/plugin-pack"
+  - "@yarnpkg/plugin-patch"
+  - "@yarnpkg/plugin-pnp"
+  - "@yarnpkg/plugin-pnpm"
+  - "@yarnpkg/plugin-stage"
+  - "@yarnpkg/plugin-typescript"
+  - "@yarnpkg/plugin-version"
+  - "@yarnpkg/plugin-workspace-tools"
+  - "@yarnpkg/builder"
+  - "@yarnpkg/doctor"
+  - "@yarnpkg/extensions"
+  - "@yarnpkg/nm"
+  - "@yarnpkg/pnpify"
+  - "@yarnpkg/sdks"

packages/docusaurus/static/configuration/yarnrc.json

@@ -1,7 +1,7 @@
 {
   "title": "JSON Schema for Yarnrc files",
   "$schema": "https://json-schema.org/draft/2019-09/schema#",
-  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). **Starting from the v2, they must be written in valid Yaml and have the right extension** (simply calling your file `.yarnrc` won't do).\n\nEnvironment variables can be accessed from setting definitions by using the `${NAME}` syntax when defining the values. By default Yarn will require the variables to be present, but this can be turned off by using either `${NAME-fallback}` (which will return `fallback` if `NAME` isn't set) or `${NAME:-fallback}` (which will return `fallback` if `NAME` isn't set, or is an empty string).\n\nFinally, note that most settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
+  "description": "Yarnrc files (named this way because they must be called `.yarnrc.yml`) are the one place where you'll be able to configure Yarn's internal settings. While Yarn will automatically find them in the parent directories, they should usually be kept at the root of your project (often your repository). **Starting from the v2, they must be written in valid Yaml and have the right extension** (simply calling your file `.yarnrc` won't do).\n\nEnvironment variable expansion is available in the following forms:\n- `${NAME}` expands to the value of the variable `NAME` and throws if it is not set\n- `${NAME:-fallback}` expands to the value of `NAME` if it is set and not empty, or the expansion of `fallback` otherwise\n- `${NAME-fallback}` expands to the value of `NAME` if it is set, or the expansion of `fallback` otherwise\n\n`$`, `\\`, and `}` can be escaped by preceding them with a `\\`. All other `\\`s and unmatched `}` are treated literally. Unclosed expansions and unescaped `${` sequences that are not recognized as an expansion throw errors.\n\nFinally, note that most settings can also be defined through environment variables (at least for the simpler ones; arrays and objects aren't supported yet). To do this, just prefix the names and write them in snake case: `YARN_CACHE_FOLDER` will set the cache folder (such values will overwrite any that might have been defined in the RC files - use them sparingly).",
   "__info": [
     "This file contains the JSON Schema for Yarnrc files and is:",
     "1) Hosted on the Yarn Website at http://yarnpkg.com/configuration/yarnrc.json",

packages/yarnpkg-core/sources/miscUtils.ts

@@ -470,26 +470,65 @@ export function buildIgnorePattern(ignorePatterns: Array<string>) {
   }).join(`|`);
 }
 
-export function replaceEnvVariables(value: string, {env}: {env: {[key: string]: string | undefined}}) {
-  const regex = /\\?\${(?<variableName>[\d\w_]+)(?<colon>:)?(?:-(?<fallback>[^}]*))?}/g;
-
-  return value.replace(regex, (match, ...args) => {
-    if (match.startsWith(`\\`))
-      return match.slice(1);
-
-    const {variableName, colon, fallback} = args[args.length - 1];
-    const variableExist = Object.hasOwn(env, variableName);
-    const variableValue = env[variableName];
+export function replaceEnvVariables(input: string, {env}: {env: {[key: string]: string | undefined}}) {
+  let output = ``;
+  let current = 0;
+  let depth = 0;
+
+  const iterator = input.matchAll(/\\(?<escaped>[\\$}])|\$\{(?<variable>[a-zA-Z]\w*)(?<operator>:-|-|(?=\}))|(?<unknown>\$\{)|\}/g);
+  const skip = () => {
+    const limit = depth;
+    for (const {0: match, index, groups: {variable}  = {}} of iterator) {
+      if (variable) {
+        depth++;
+      } else if (match === `}`) {
+        if (--depth < limit) {
+          return index + match.length;
+        }
+      }
+    }
+    return input.length;
+  };
+
+  for (const {0: match, index, groups: {escaped, variable, operator, unknown} = {}} of iterator) {
+    output += input.slice(current, index);
+    current = index + match.length;
+
+    if (escaped) {
+      output += escaped;
+    } else if (variable) {
+      const value = env[variable];
+      depth++;
+
+      if (
+        // ${VAR}
+        (operator === `` && value !== undefined) ||
+        // ${VAR:-
+        (operator === `:-` && value !== undefined && value !== ``) ||
+        // ${VAR-
+        (operator === `-` && value !== undefined)
+      ) {
+        output += value;
+        current = skip();
+      } else if (operator === ``) {
+        // ${NON_EXISTENT}
+        throw new UsageError(`Environment variable not found (${variable})`);
+      }
+    } else if (match === `}`) {
+      if (depth === 0) {
+        output += match;
+      } else {
+        depth--;
+      }
+    } else if (unknown) {
+      throw new UsageError(`Invalid environment variable substitution syntax: ${input}`);
+    }
+  }
 
-    if (variableValue)
-      return variableValue;
-    if (variableExist && !colon)
-      return variableValue;
-    if (fallback != null)
-      return fallback;
+  if (depth > 0)
+    throw new UsageError(`Incomplete variable substitution in input: ${input}`);
 
-    throw new UsageError(`Environment variable not found (${variableName})`);
-  });
+  return output + input.slice(current);
 }
 
 export function parseBoolean(value: unknown): boolean {

packages/yarnpkg-core/tests/Configuration.test.ts

@@ -114,6 +114,27 @@ describe(`Configuration`, () => {
           emptyEnvWithEmptyFallback: {
             npmAuthToken: `\${EMPTY_VARIABLE:-}`,
           },
+          emptyEnvWithNestedEnv: {
+            npmAuthToken: `prefix1-\${EMPTY_VARIABLE:-prefix2-\${ENV_AUTH_TOKEN-fallback-value}-suffix1}-suffix2`,
+          },
+          emptyEnvWithNestedEnvWithStrictFallback: {
+            npmAuthToken: `prefix1-\${EMPTY_VARIABLE:-prefix2-\${EMPTY_VARIABLE-fallback-value}-suffix1}-suffix2`,
+          },
+          emptyEnvWithNestedEnvWithFallback: {
+            npmAuthToken: `prefix1-\${EMPTY_VARIABLE:-prefix2-\${EMPTY_VARIABLE:-fallback-value}-suffix1}-suffix2`,
+          },
+          escapedEnv: {
+            npmAuthToken: `\\\${ENV_AUTH_TOKEN}`,
+          },
+          escapedBackslash: {
+            npmAuthToken: `\\\\\${ENV_AUTH_TOKEN}`,
+          },
+          escapedBrace: {
+            npmAuthToken: `\${EMPTY_VARIABLE:-\\}}`,
+          },
+          literalBrace: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN}}`,
+          },
         },
       }, async dir => {
         const configuration = await Configuration.find(dir, {
@@ -132,6 +153,13 @@ describe(`Configuration`, () => {
         const emptyEnvWithStrictFallback = getToken(`emptyEnvWithStrictFallback`);
         const emptyEnvWithFallback = getToken(`emptyEnvWithFallback`);
         const emptyEnvWithEmptyFallback = getToken(`emptyEnvWithEmptyFallback`);
+        const emptyEnvWithNestedEnv = getToken(`emptyEnvWithNestedEnv`);
+        const emptyEnvWithNestedEnvWithStrictFallback = getToken(`emptyEnvWithNestedEnvWithStrictFallback`);
+        const emptyEnvWithNestedEnvWithFallback = getToken(`emptyEnvWithNestedEnvWithFallback`);
+        const escapedEnv = getToken(`escapedEnv`);
+        const escapedBackslash = getToken(`escapedBackslash`);
+        const escapedBrace = getToken(`escapedBrace`);
+        const literalBrace = getToken(`literalBrace`);
 
         expect(onlyEnv).toEqual(`AAA-BBB-CCC`);
         expect(multipleEnvs).toEqual(`AAA-BBB-CCC-separator-AAA-BBB-CCC`);
@@ -142,6 +170,13 @@ describe(`Configuration`, () => {
         expect(emptyEnvWithStrictFallback).toEqual(``);
         expect(emptyEnvWithFallback).toEqual(`fallback-for-empty-value`);
         expect(emptyEnvWithEmptyFallback).toEqual(``);
+        expect(emptyEnvWithNestedEnv).toEqual(`prefix1-prefix2-AAA-BBB-CCC-suffix1-suffix2`);
+        expect(emptyEnvWithNestedEnvWithStrictFallback).toEqual(`prefix1-prefix2--suffix1-suffix2`);
+        expect(emptyEnvWithNestedEnvWithFallback).toEqual(`prefix1-prefix2-fallback-value-suffix1-suffix2`);
+        expect(escapedEnv).toEqual(`\${ENV_AUTH_TOKEN}`);
+        expect(escapedBackslash).toEqual(`\\AAA-BBB-CCC`);
+        expect(escapedBrace).toEqual(`}`);
+        expect(literalBrace).toEqual(`AAA-BBB-CCC}`);
       });
     });
 
@@ -160,6 +195,83 @@ describe(`Configuration`, () => {
       });
     });
 
+    it(`should allow unset variables in unused fallbacks`, async () => {
+      process.env.ENV_AUTH_TOKEN = `AAA-BBB-CCC`;
+
+      await initializeConfiguration({
+        npmScopes: {
+          onlyEnv: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN:-\${A_VARIABLE_THAT_DEFINITELY_DOESNT_EXIST}}`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).resolves.toBeDefined();
+      });
+    });
+
+    it(`should forbid unclosed variable substitution`, async () => {
+      process.env.ENV_AUTH_TOKEN = `AAA-BBB-CCC`;
+
+      await initializeConfiguration({
+        npmScopes: {
+          noFallback: {
+            npmAuthToken: `\${A_VARIABLE_THAT_DEFINITELY_DOESNT_EXIST`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).rejects.toThrow();
+      });
+
+      await initializeConfiguration({
+        npmScopes: {
+          withFallback: {
+            npmAuthToken: `\${A_VARIABLE_THAT_DEFINITELY_DOESNT_EXIST:-`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).rejects.toThrow();
+      });
+
+      await initializeConfiguration({
+        npmScopes: {
+          withNestedFallback: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN:-\${A_VARIABLE_THAT_DEFINITELY_DOESNT_EXIST:-`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).rejects.toThrow();
+      });
+    });
+
+    it(`should forbid unknown operators`, async () => {
+      process.env.ENV_AUTH_TOKEN = `AAA-BBB-CCC`;
+
+      await initializeConfiguration({
+        npmScopes: {
+          onlyEnv: {
+            npmAuthToken: `\${ENV_AUTH_TOKEN:+}`,
+          },
+        },
+      }, async dir => {
+        await expect(Configuration.find(dir, {
+          modules: new Map([[`@yarnpkg/plugin-npm`, NpmPlugin]]),
+          plugins: new Set([`@yarnpkg/plugin-npm`]),
+        })).rejects.toThrow();
+      });
+    });
+
     it(`should handle boolean variables correctly`, async () => {
       process.env.TRUE_VARIABLE = `true`;
       process.env.FALSE_VARIABLE = `false`;