From 724f9213669dbb7c73871763f1f00624716616ee Mon Sep 17 00:00:00 2001 From: James Arthur Date: Mon, 3 Oct 2022 16:39:29 +0200 Subject: [PATCH] Update docs + local dev fixes. (#47) * config: configure `SQConnectors` at runtime in prod. * docs: update README and remove legacy TODO. * Updates the README with new brand and better instructions. * Update config/runtime.exs Co-authored-by: Ilia Borovitinov --- README.md | 118 ++++++++++++++++++++++----------------------- TODO.md | 26 ---------- config/runtime.exs | 5 ++ 3 files changed, 62 insertions(+), 87 deletions(-) delete mode 100644 TODO.md diff --git a/README.md b/README.md index 5301eb09..7e7fd5f9 100644 --- a/README.md +++ b/README.md @@ -1,87 +1,83 @@ + + + + + ElectricSQL logo + + # Electric -This is a POC of Postgres active-active replication using Vaxine. +This is the replication service for [ElectricSQL](https://electric-sql.com). + +It's an Elixir application that integrates with Postgres over logical replication and Satellite (ElectricSQL's client-side replication component that works with SQLite) via a web socket interface. ## Pre-reqs Docker and Elixir 1.13. -## Run databases +## Usage + +See the [Makefile](./Makefile) for usage. Setup using: ```sh -docker-compose -f databases.yaml up +make deps compile ``` -## Run app +Run the dependencies using: ```sh -mix run --no-halt +make start_dev_env ``` -## Generate workload +Run the tests: + +```sh +make tests +``` -For now, manually issue some SQL statements, e.g.: +And then develop using: +```sh +make shell ``` -psql -h localhost -p 54321 -U electric -d electric -... -electric=# INSERT INTO entries (content) VALUES ('a'); -electric=# select * from entries; -electric=# update entries set content = 'b'; + +This runs active-active replication with Postgres over logical replication and exposes a protocol buffers API over web sockets on `localhost:30002` for the ElectricSQL client libraries. + +For example to write some data into one of the Postgres instances: + +```sh +docker exec -it -e PGPASSWORD=password electric_db_a_1 psql -h 127.0.0.1 -U electric -d electric ``` -Then view the app logs, should look a bit like: +There's a second instance, `electric-db_b_1`, if you want to see data being replicated between them. + +## OSX + +Note that if, when running on OSX, you get errors like: ``` -{:message, - %Broadway.Message{ - acknowledger: {Electric.Replication, :ack_id, {#PID<0.218.0>, {0, 24336352}}}, - batch_key: :default, - batch_mode: :bulk, - batcher: :default, - data: %Electric.Replication.Changes.Transaction{ - changes: [ - %Electric.Replication.Changes.NewRecord{ - record: %{ - "content" => "a", - "id" => "9be3b616-17e9-4264-9f33-5bdb36c48443" - }, - relation: {"public", "entries"} - } - ], - commit_timestamp: ~U[2022-06-01 14:07:56Z] - }, - metadata: %{}, - status: :ok - }} -{:ack, {0, 24336352}} -{:message, - %Broadway.Message{ - acknowledger: {Electric.Replication, :ack_id, {#PID<0.218.0>, {0, 24336568}}}, - batch_key: :default, - batch_mode: :bulk, - batcher: :default, - data: %Electric.Replication.Changes.Transaction{ - changes: [ - %Electric.Replication.Changes.UpdatedRecord{ - old_record: %{ - "content" => "a", - "id" => "9be3b616-17e9-4264-9f33-5bdb36c48443" - }, - record: %{ - "content" => "b", - "id" => "9be3b616-17e9-4264-9f33-5bdb36c48443" - }, - relation: {"public", "entries"} - } - ], - commit_timestamp: ~U[2022-06-01 14:08:39Z] - }, - metadata: %{}, - status: :ok - }} +could not connect to the publisher: connection to server at \"host.docker.internal\" (192.168.65.2), port 5433 failed ``` -Note the `old_record` as well as the `new_record`. +You may need to adjust your docker networking or run Electric within docker. To run within Docker, you can build the docker image locally: +```sh +make docker-build +``` + +And then run with the right env vars, e.g.: + +```sh +docker run -it -p "5433:5433" -p "5133:5133" \ + -e "VAXINE_HOSTNAME=host.docker.internal" + -e "ELECTRIC_HOST=host.docker.internal" + -e "CONNECTORS=pg1=postgresql://electric:password@host.docker.internal:54321/electric;pg2=postgresql://electric:password@host.docker.internal:54322/electric" \ + docker.io/library/electric:local-build +``` diff --git a/TODO.md b/TODO.md deleted file mode 100644 index c79cc9cc..00000000 --- a/TODO.md +++ /dev/null @@ -1,26 +0,0 @@ -# Todo - -Step 1: - -- [x] two Postgres database nodes (same single table schema for now) -- [x] setup logical replication between them - -Step 2: - -- [x] consume and decode logical replication stream from Postgres A -- [ ] implement enough of the backend server logical replication protocol in order to replicate through Elixir -- [ ] encode and produce logical replication stream for Postgres B - -Step 3: - -- [ ] write changes into Antidote -- [ ] somehow handle `LSN` polling / pull from Postgres B -- [ ] query relevant materialised values from Antidote -- [ ] construct into encodable stream - -Step 4: - -- [ ] add a third Postgres -- [ ] pair each Postgres with an Antidote -- [ ] replicate between Antidotes -- [ ] demonstrate Postgres replication working with TCC+ diff --git a/config/runtime.exs b/config/runtime.exs index 31af68dd..0181872a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -63,4 +63,9 @@ if config_env() == :prod do end) config :electric, Electric.Replication.Connectors, connectors + + config :electric, Electric.Replication.SQConnectors, + vaxine_hostname: vaxine_hostname, + vaxine_port: 8088, + vaxine_connection_timeout: vaxine_connection_timeout end