From 3f3f6f2993d8404a015854981bf4f597544bd422 Mon Sep 17 00:00:00 2001 From: Ilia Borovitinov Date: Wed, 23 Nov 2022 17:30:58 +0300 Subject: [PATCH] chore: streamlined environment variable usage & documentation (#68) - Makes migration dir & file suffix more configurable - Makes sure we're calling `System.get_env` or `fetch_env` only in `runtime.exs` - Adds documentation for all current environment variables, apart from auth since it's changed in another open PR - Moves out offset storage configuration from compile-time to runtime - Updated environment variable names for consistency --- .gitignore | 4 +-- Dockerfile | 2 +- Makefile | 2 +- README.md | 28 ++++++++++++++++-- config/dev.exs | 6 ++-- config/prod.exs | 2 +- config/runtime.exs | 33 ++++++++++++++++------ config/test.exs | 4 ++- lib/electric/migration_utils.ex | 21 ++++++-------- lib/electric/replication/offset_storage.ex | 11 ++++---- 10 files changed, 75 insertions(+), 38 deletions(-) diff --git a/.gitignore b/.gitignore index e3826992..586b1e28 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,4 @@ electric-*.tar # Temporary files, for example, from tests. /tmp/ -vx_pg_offset_storage_dev.dat -vx_pg_offset_storage_test.dat -vx_pg_offset_storage_prod.dat +*offset_storage*.dat diff --git a/Dockerfile b/Dockerfile index 96b45eef..ed8f405d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ FROM runner_setup AS runner ## Vaxine configuration via environment variables COPY --from=builder --chown=nobody:root /app/_build/prod/rel/electric ./ -VOLUME ./vx_pg_offset_storage_prod.dat +VOLUME ./offset_storage_data.dat USER nobody ENTRYPOINT /app/bin/electric start diff --git a/Makefile b/Makefile index 1f9878db..c7c02198 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ ifneq ($(docker images -q electric:local-build 2> /dev/null), "") endif rm_offset_storage: - rm vx_pg_offset_storage_* + rm offset_storage_* update_protobuf: deps mix protox.generate \ diff --git a/README.md b/README.md index c82f0616..259420e5 100644 --- a/README.md +++ b/README.md @@ -67,12 +67,34 @@ Note that you can tear down all the containers with: make stop_dev_env ``` +### Running the release or docker container + +The Electric application is configured using environment variables. Everything that doesn't have a default is required to run. + +| Variable | Default | Description | +| --- | --- | --- | +| `VAXINE_HOST` | | Host of Vaxine instance to connect to | +| `VAXINE_API_PORT` | `8087` | Port for the regular DB API on Vaxine instance | +| `VAXINE_REPLICATION_PORT` | `8088` | Port for the replication API on Vaxine instance | +| `VAXINE_CONNECTION_TIMEOUT` | `5000` | (ms) Timeout waiting while connecting to a Vaxine instance | +| | +| `ELECTRIC_HOST` | | Host of this electric instance for the reverse connection from Postgres. It has to be accessible from postgres instances listed in the `CONNECTORS` | +| `CONNECTORS` | `""` | Semicolon-separated list of Postgres connection strings for PG instances that will be part of the cluster | +| | +| `POSTGRES_REPLICATION_PORT` | `5433` | Port for connections from PG instances as replication followers | +| `STATUS_PORT` | `5050` | Port to expose health and status API endpoint | +| `WEBSOCKET_PORT` | `5133` | Port to expose the `/ws` path for the replication over the websocket | +| | +| `OFFSET_STORAGE_FILE` | `./offset_storage_data.dat` | Path to the file storing the mapping between connected instances and offsets in Vaxine WAL. Should be persisted between Electric restarts. | +| `MIGRATIONS_DIR` | | Directory to read the migration SQL files from | +| `MIGRATIONS_FILE_NAME_SUFFIX` | `/postgres.sql` | Suffix that is appended to the migration name when looking for the migration file | + ## Migrations -When running locally, you can apply migrations directly using `make apply_migration`. First make sure you've [built your migrations](https://electric-sql.com/docs/usage/migrations) in your application folder, then set the `ELECTRIC_MIGRATIONS_DIR` environment variable to the path to the migrations folder: +When running locally, you can apply migrations directly using `make apply_migration`. First make sure you've [built your migrations](https://electric-sql.com/docs/usage/migrations) in your application folder, then set the `MIGRATIONS_DIR` environment variable to the path to the migrations folder: ```sh -export ELECTRIC_MIGRATIONS_DIR='../path/to/migrations' +export MIGRATIONS_DIR='../path/to/migrations' ``` Now (re)run the electric service (with the env var set): @@ -111,7 +133,7 @@ And then run with the right env vars, e.g.: ```sh docker run -it -p "5433:5433" -p "5133:5133" \ - -e "VAXINE_HOSTNAME=host.docker.internal" + -e "VAXINE_HOST=host.docker.internal" -e "ELECTRIC_HOST=host.docker.internal" -e "CONNECTORS=pg1=postgresql://electric:password@host.docker.internal:54321/electric;pg2=postgresql://electric:password@host.docker.internal:54322/electric" \ docker.io/library/electric:local-build diff --git a/config/dev.exs b/config/dev.exs index d23d9ca0..f7ed589e 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -2,9 +2,11 @@ import Config config :electric, Electric.VaxRepo, hostname: "localhost", port: 8087 -config :electric, Electric.Replication.OffsetStorage, file: "./vx_pg_offset_storage_dev.dat" +config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.dev.dat" -config :electric, Electric.Migrations, dir: "./integration_tests/migrations/migration_schemas/" +config :electric, Electric.Migrations, + migration_file_name_suffix: "/postgres.sql", + dir: System.get_env("MIGRATIONS_DIR", "./integration_tests/migrations/migration_schemas/") config :electric, Electric.Replication.Connectors, postgres_1: [ diff --git a/config/prod.exs b/config/prod.exs index 2dbaaa5f..a4774b2f 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,6 +1,6 @@ import Config -config :electric, Electric.Replication.OffsetStorage, file: "./vx_pg_offset_storage_prod.dat" +config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.prod.dat" # Do not print debug messages in production config :logger, level: :info diff --git a/config/runtime.exs b/config/runtime.exs index 29128a36..7eda41b4 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -11,20 +11,30 @@ if config_env() == :prod do config :electric, Electric.StatusPlug, port: System.get_env("STATUS_PORT", "5050") |> String.to_integer() - vaxine_hostname = - System.get_env("VAXINE_HOSTNAME") || raise "Env variable VAXINE_HOSTNAME is not set" + vaxine_hostname = System.get_env("VAXINE_HOST") || raise "Env variable VAXINE_HOST is not set" vaxine_connection_timeout = System.get_env("VAXINE_CONNECTION_TIMEOUT", "5000") |> String.to_integer() + vaxine_antidote_port = System.get_env("VAXINE_API_PORT", "8087") |> String.to_integer() + + vaxine_replication_port = + System.get_env("VAXINE_REPLICATION_PORT", "8088") |> String.to_integer() + config :electric, Electric.VaxRepo, hostname: vaxine_hostname, - port: 8087 + port: vaxine_antidote_port - publication = System.get_env("PUBLICATION", "all_tables") - slot = System.get_env("SLOT", "all_changes") + publication = System.get_env("POSTGRES_PUBLICATION", "all_tables") + slot = System.get_env("POSTGRES_SLOT", "all_changes") electric_host = System.get_env("ELECTRIC_HOST") || raise "Env variable ELECTRIC_HOST is not set" - electric_port = System.get_env("ELECTRIC_PORT", "5433") |> String.to_integer() + + electric_port = System.get_env("POSTGRES_REPLICATION_PORT", "5433") |> String.to_integer() + + config :electric, Electric.PostgresServer, port: electric_port + + config :electric, Electric.Satellite.WsServer, + port: System.get_env("WEBSOCKET_PORT", "5133") |> String.to_integer() connectors = System.get_env("CONNECTORS", "") @@ -56,7 +66,7 @@ if config_env() == :prod do producer: Electric.Replication.Vaxine.LogProducer, producer_opts: [ vaxine_hostname: vaxine_hostname, - vaxine_port: 8088, + vaxine_port: vaxine_replication_port, vaxine_connection_timeout: vaxine_connection_timeout ] ]} @@ -66,9 +76,16 @@ if config_env() == :prod do config :electric, Electric.Replication.SQConnectors, vaxine_hostname: vaxine_hostname, - vaxine_port: 8088, + vaxine_port: vaxine_replication_port, vaxine_connection_timeout: vaxine_connection_timeout + config :electric, Electric.Replication.OffsetStorage, + file: System.get_env("OFFSET_STORAGE_FILE", "./offset_storage_data.dat") + + config :electric, Electric.Migrations, + dir: System.fetch_env!("MIGRATIONS_DIR"), + migration_file_name_suffix: System.get_env("MIGRATIONS_FILE_NAME_SUFFIX", "/postgres.sql") + # set to the database.cluster_slug global_cluster_id = System.fetch_env!("GLOBAL_CLUSTER_ID") diff --git a/config/test.exs b/config/test.exs index 674bec3f..ed21082a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -1,6 +1,6 @@ import Config -config :electric, Electric.Replication.OffsetStorage, file: "./vx_pg_offset_storage_test.dat" +config :electric, Electric.Replication.OffsetStorage, file: "./offset_storage_data.test.dat" config :electric, Electric.VaxRepo, hostname: "localhost", port: 8087 @@ -27,6 +27,8 @@ config :electric, Electric.Replication.SQConnectors, vaxine_port: 8088, vaxine_connection_timeout: 5000 +config :electric, Electric.Migrations, migration_file_name_suffix: "/postgres.sql" + config :electric, global_cluster_id: "electric-development-cluster-0000" config :electric, Electric.Satellite.Auth, provider: {Electric.Satellite.Auth.Insecure, []} diff --git a/lib/electric/migration_utils.ex b/lib/electric/migration_utils.ex index 8bf0264f..a3ff494a 100644 --- a/lib/electric/migration_utils.ex +++ b/lib/electric/migration_utils.ex @@ -16,20 +16,15 @@ defmodule Electric.Migration.Utils do @spec get_migration_path(vsn()) :: binary def get_migration_path(vsn) do - migration_path = get_migration_dir() - Path.join(migration_path, vsn <> "/postgres.sql") + migration_dir = fetch_config!(:dir) + file_name_suffix = fetch_config!(:migration_file_name_suffix) + Path.join(migration_dir, vsn <> file_name_suffix) end - defp get_migration_dir() do - case :os.getenv(to_charlist("ELECTRIC_MIGRATIONS_DIR")) do - false -> - Keyword.fetch!( - Application.get_env(:electric, Electric.Migrations), - :dir - ) - - value -> - value - end + defp fetch_config!(key) when key in [:dir, :migration_file_name_suffix] do + Keyword.fetch!( + Application.get_env(:electric, Electric.Migrations), + key + ) end end diff --git a/lib/electric/replication/offset_storage.ex b/lib/electric/replication/offset_storage.ex index bbd4a426..ad247b86 100644 --- a/lib/electric/replication/offset_storage.ex +++ b/lib/electric/replication/offset_storage.ex @@ -12,16 +12,17 @@ defmodule Electric.Replication.OffsetStorage do @table Module.concat([__MODULE__, Table]) - @default_file Application.compile_env!(:electric, __MODULE__) - |> Keyword.fetch!(:file) - |> String.to_charlist() - def start_link(opts) do GenServer.start_link(__MODULE__, opts) end def init(opts) do - opts = Keyword.merge([file: @default_file, type: :set], opts) + default_file = Application.fetch_env!(:electric, __MODULE__) |> Keyword.fetch!(:file) + + opts = + Keyword.merge([file: default_file, type: :set], opts) + |> Keyword.update!(:file, &String.to_charlist/1) + dets = :dets.open_file(@table, opts) {:ok, dets}