add dotenv support to PlatformConfigProvider (#3743)

Co-authored-by: Tim <[email protected]>

.changeset/neat-olives-learn.md

@@ -0,0 +1,24 @@
+---
+"@effect/platform": patch
+---
+
+Add support for `ConfigProvider` based on .env files.
+
+```ts
+import { PlatformConfigProvider } from "@effect/platform"
+import { NodeContext } from "@effect/platform-node"
+import { Config } from "effect"
+
+Effect.gen(function* () {
+  const config = yield* Config.all({
+    api_url: Config.string("API_URL"),
+    api_key: Config.string("API_KEY")
+  })
+
+  console.log(`Api config: ${config}`)
+}).pipe(
+  Effect.provide(PlatformConfigProvider.layerDotEnvAdd(".env").pipe(
+    Layer.provide(NodeContext.layer)
+  )),
+)
+```

packages/platform-node/test/PlatformConfigProvider.test.ts

@@ -0,0 +1,113 @@
+import { FileSystem, Path, PlatformConfigProvider } from "@effect/platform"
+import { NodeContext } from "@effect/platform-node"
+import { describe, expect, it } from "@effect/vitest"
+import { Config, Effect, Either } from "effect"
+
+describe("dotenv", () => {
+  const ExampleConfig = Config.all({
+    value: Config.string("VALUE"),
+    number: Config.number("NUMBER")
+  })
+
+  it.scopedLive.each([
+    {
+      name: "Simple variables",
+      config: ExampleConfig,
+      content: "VALUE=hello\nNUMBER=69",
+      expected: { value: "hello", number: 69 }
+    },
+    {
+      name: "Whitespaces",
+      config: ExampleConfig,
+      content: "VALUE= hello  \n NUMBER= 69 \n\n",
+      expected: { value: "hello", number: 69 }
+    },
+    {
+      name: "Quotes",
+      config: Config.all({
+        value: Config.string("VALUE"),
+        anotherValue: Config.string("ANOTHER_VALUE")
+      }),
+      content: "VALUE=\" hello  \"\nANOTHER_VALUE=' another   '",
+      expected: { value: " hello  ", anotherValue: " another   " }
+    },
+    {
+      name: "Expand",
+      config: ExampleConfig,
+      content: "VALUE=hello-${NUMBER}\nNUMBER=69",
+      expected: { value: "hello-69", number: 69 }
+    }
+  ])("parsing ($name)", ({ config, content, expected }) =>
+    Effect.gen(function*() {
+      const envFile = yield* createTmpEnvFile(content)
+      const result = yield* (config as Config.Config<unknown>).pipe(
+        Effect.provide(PlatformConfigProvider.layerDotEnv(envFile))
+      )
+      expect(result).toEqual(expected)
+    }).pipe(Effect.provide(NodeContext.layer)))
+
+  it.scopedLive("load from both process env and dotenv file", () =>
+    Effect.gen(function*() {
+      yield* modifyEnv("VALUE", "hello")
+      const envFile = yield* createTmpEnvFile("NUMBER=69")
+      const result = yield* ExampleConfig.pipe(
+        Effect.provide(PlatformConfigProvider.layerDotEnvAdd(envFile))
+      )
+      expect(result).toEqual({ value: "hello", number: 69 })
+    }).pipe(Effect.provide(NodeContext.layer)))
+
+  it.scopedLive("current ConfigProvider has precedence over dotenv", () =>
+    Effect.gen(function*() {
+      yield* modifyEnv("VALUE", "hello")
+      const envFile = yield* createTmpEnvFile("NUMBER=69\nVALUE=another")
+      const result = yield* ExampleConfig.pipe(
+        Effect.provide(PlatformConfigProvider.layerDotEnvAdd(envFile))
+      )
+      expect(result).toEqual({ value: "hello", number: 69 })
+    }).pipe(Effect.provide(NodeContext.layer)))
+
+  it.scopedLive("fromDotEnv fails if no .env file is found", () =>
+    Effect.gen(function*() {
+      const result = yield* PlatformConfigProvider.fromDotEnv(".non-existing-env-file").pipe(Effect.either)
+      expect(Either.isLeft(result)).toBe(true)
+    }).pipe(Effect.provide(NodeContext.layer)))
+
+  it.scopedLive("layerDotEnvAdd succeeds if no .env file is found", () =>
+    Effect.gen(function*() {
+      yield* modifyEnv("VALUE", "hello")
+      const value = yield* Config.string("VALUE")
+      expect(value).toEqual("hello")
+    }).pipe(
+      Effect.provide(PlatformConfigProvider.layerDotEnvAdd(".non-existing-env-file")),
+      Effect.provide(NodeContext.layer)
+    ))
+})
+
+// utils
+
+const createTmpEnvFile = (data: string) =>
+  Effect.gen(function*() {
+    const fs = yield* FileSystem.FileSystem
+    const path = yield* Path.Path
+    const dir = yield* fs.makeTempDirectoryScoped({ prefix: "tmp" })
+    const filename = path.join(dir, ".env")
+    yield* fs.writeFileString(filename, data)
+    return filename
+  })
+
+const modifyEnv = (key: string, value: string) =>
+  Effect.gen(function*() {
+    const isInEnv = key in process.env
+    const original = process.env[key]
+    process.env[key] = value
+
+    yield* Effect.addFinalizer(() =>
+      Effect.sync(() => {
+        if (isInEnv) {
+          process.env[key] = original
+        } else {
+          delete process.env[key]
+        }
+      })
+    )
+  })

packages/platform/src/PlatformConfigProvider.ts

@@ -15,6 +15,7 @@ import * as HashSet from "effect/HashSet"
 import * as Layer from "effect/Layer"
 import { isPlatformError, type PlatformError } from "./Error.js"
 import * as FileSystem from "./FileSystem.js"
+import * as internal from "./internal/platformConfigProvider.js"
 import * as Path from "./Path.js"
 
 /**
@@ -111,3 +112,32 @@ export const layerFileTree = (options?: {
     Effect.map(Layer.setConfigProvider),
     Layer.unwrapEffect
   )
+
+/**
+ * Create a dotenv ConfigProvider.
+ *
+ * @category constructors
+ * @since 1.0.0
+ */
+export const fromDotEnv: (
+  paths: string
+) => Effect.Effect<ConfigProvider.ConfigProvider, PlatformError, FileSystem.FileSystem> = internal.fromDotEnv
+
+/**
+ * Add the dotenv ConfigProvider to the environment, as a fallback to the current ConfigProvider.
+ * If the file is not found, a debug log is produced and empty layer is returned.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerDotEnvAdd: (path: string) => Layer.Layer<never, never, FileSystem.FileSystem> =
+  internal.layerDotEnvAdd
+
+/**
+ * Add the dotenv ConfigProvider to the environment, replacing the current ConfigProvider.
+ *
+ * @since 1.0.0
+ * @category layers
+ */
+export const layerDotEnv: (path: string) => Layer.Layer<never, PlatformError, FileSystem.FileSystem> =
+  internal.layerDotEnv

packages/platform/src/internal/platformConfigProvider.ts

@@ -0,0 +1,148 @@
+import type { PlatformError } from "@effect/platform/Error"
+import * as FileSystem from "@effect/platform/FileSystem"
+import * as ConfigProvider from "effect/ConfigProvider"
+import * as Context from "effect/Context"
+import * as DefaultServices from "effect/DefaultServices"
+import * as Effect from "effect/Effect"
+import * as FiberRef from "effect/FiberRef"
+import * as Layer from "effect/Layer"
+
+/**
+ * dot env ConfigProvider
+ *
+ * Based on
+ * - https://github.com/motdotla/dotenv
+ * - https://github.com/motdotla/dotenv-expand
+ */
+
+/** @internal */
+export const fromDotEnv = (
+  path: string
+): Effect.Effect<ConfigProvider.ConfigProvider, PlatformError, FileSystem.FileSystem> =>
+  Effect.gen(function*(_) {
+    const fs = yield* FileSystem.FileSystem
+    const content = yield* fs.readFileString(path)
+    const obj = parseDotEnv(content)
+    return ConfigProvider.fromJson(obj)
+  })
+
+/** @internal */
+export const layerDotEnv = (path: string): Layer.Layer<never, PlatformError, FileSystem.FileSystem> =>
+  fromDotEnv(path).pipe(
+    Effect.map(Layer.setConfigProvider),
+    Layer.unwrapEffect
+  )
+
+/** @internal */
+export const layerDotEnvAdd = (path: string): Layer.Layer<never, never, FileSystem.FileSystem> =>
+  Effect.gen(function*(_) {
+    const dotEnvConfigProvider = yield* Effect.orElseSucceed(fromDotEnv(path), () => null)
+
+    if (dotEnvConfigProvider === null) {
+      yield* Effect.logDebug(`File '${path}' not found, skipping dotenv ConfigProvider.`)
+      return Layer.empty
+    }
+
+    const currentConfigProvider = yield* FiberRef.get(DefaultServices.currentServices).pipe(
+      Effect.map((services) => Context.get(services, ConfigProvider.ConfigProvider))
+    )
+    const configProvider = ConfigProvider.orElse(currentConfigProvider, () => dotEnvConfigProvider)
+    return Layer.setConfigProvider(configProvider)
+  }).pipe(Layer.unwrapEffect)
+
+/** @internal */
+const DOT_ENV_LINE =
+  /(?:^|^)\s*(?:export\s+)?([\w.-]+)(?:\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?(?:$|$)/mg
+
+/** @internal */
+const parseDotEnv = (lines: string): Record<string, string> => {
+  const obj: Record<string, string> = {}
+
+  // Convert line breaks to same format
+  lines = lines.replace(/\r\n?/gm, "\n")
+
+  let match: RegExpExecArray | null
+  while ((match = DOT_ENV_LINE.exec(lines)) != null) {
+    const key = match[1]
+
+    // Default undefined or null to empty string
+    let value = match[2] || ""
+
+    // Remove whitespace
+    value = value.trim()
+
+    // Check if double quoted
+    const maybeQuote = value[0]
+
+    // Remove surrounding quotes
+    value = value.replace(/^(['"`])([\s\S]*)\1$/gm, "$2")
+
+    // Expand newlines if double quoted
+    if (maybeQuote === "\"") {
+      value = value.replace(/\\n/g, "\n")
+      value = value.replace(/\\r/g, "\r")
+    }
+
+    // Add to object
+    obj[key] = value
+  }
+
+  return expand(obj)
+}
+
+/** @internal */
+const expand = (parsed: Record<string, string>) => {
+  const newParsed: Record<string, string> = {}
+
+  for (const configKey in parsed) {
+    // resolve escape sequences
+    newParsed[configKey] = interpolate(parsed[configKey], parsed).replace(/\\\$/g, "$")
+  }
+
+  return newParsed
+}
+
+/** @internal */
+const interpolate = (envValue: string, parsed: Record<string, string>) => {
+  // find the last unescaped dollar sign in the
+  // value so that we can evaluate it
+  const lastUnescapedDollarSignIndex = searchLast(envValue, /(?!(?<=\\))\$/g)
+
+  // If we couldn't match any unescaped dollar sign
+  // let's return the string as is
+  if (lastUnescapedDollarSignIndex === -1) return envValue
+
+  // This is the right-most group of variables in the string
+  const rightMostGroup = envValue.slice(lastUnescapedDollarSignIndex)
+
+  /**
+   * This finds the inner most variable/group divided
+   * by variable name and default value (if present)
+   * (
+   *   (?!(?<=\\))\$        // only match dollar signs that are not escaped
+   *   {?                   // optional opening curly brace
+   *     ([\w]+)            // match the variable name
+   *     (?::-([^}\\]*))?   // match an optional default value
+   *   }?                   // optional closing curly brace
+   * )
+   */
+  const matchGroup = /((?!(?<=\\))\${?([\w]+)(?::-([^}\\]*))?}?)/
+  const match = rightMostGroup.match(matchGroup)
+
+  if (match !== null) {
+    const [_, group, variableName, defaultValue] = match
+
+    return interpolate(
+      envValue.replace(group, defaultValue || parsed[variableName] || ""),
+      parsed
+    )
+  }
+
+  return envValue
+}
+
+/** @internal */
+const searchLast = (str: string, rgx: RegExp) => {
+  const matches = Array.from(str.matchAll(rgx))
+  return matches.length > 0 ? matches.slice(-1)[0].index : -1
+}