feat: add git_resolve function

This resolves a git ref into a json object, which can be passed to update or insert operations to restore previous versions of rows.

Author: mmkal

package.json

@@ -2,6 +2,14 @@
   "name": "plv8-git",
   "version": "0.0.1",
   "description": "Tracks history of rows in postgresql database tables, using in-memory git operations",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mmkal/plv8-git.git"
+  },
+  "homepage": "https://github.com/mmkal/plv8-git#readme",
+  "bugs": {
+    "url": "https://github.com/mmkal/plv8-git/issues"
+  },
   "keywords": [
     "postgresql",
     "git",
@@ -13,14 +21,6 @@
     "memfs",
     "event-sourcing"
   ],
-  "homepage": "https://github.com/mmkal/plv8-git#readme",
-  "bugs": {
-    "url": "https://github.com/mmkal/plv8-git/issues"
-  },
-  "repository": {
-    "type": "git",
-    "url": "https://github.com/mmkal/plv8-git.git"
-  },
   "license": "Apache-2.0",
   "main": "dist/bundle.js",
   "types": "dist/index.d.ts",
@@ -32,6 +32,8 @@
     "compile": "tsc -p tsconfig.lib.json",
     "predocker-apply": "yarn docker-copy-query",
     "docker-apply": "yarn psql -f /queries/create-git-functions.sql",
+    "docker-bash": "docker-compose exec postgres bash",
+    "docker-psql": "docker-compose exec postgres psql -h localhost -U postgres postgres",
     "predocker-copy-query": "yarn docker-exec mkdir -p /queries",
     "docker-copy-query": "docker cp queries/create-git-functions.sql plv8-git_postgres_1:/queries",
     "docker-exec": "docker exec plv8-git_postgres_1",

readme.md

@@ -10,6 +10,7 @@ The implementation uses [plv8](https://github.com/plv8/plv8) to run JavaScript i
    - [Tracking history](#tracking-history)
    - [Deletions](#deletions)
    - [Configuraton](#configuraton)
+   - [Restoring previous versions](#restoring-previous-versions)
 - [Caveat](#caveat)
 - [Implementation](#implementation)
 <!-- codegen:end -->
@@ -18,6 +19,7 @@ The implementation uses [plv8](https://github.com/plv8/plv8) to run JavaScript i
 
 To paraphrase [@mayfer's twitter thread](https://twitter.com/mayfer/status/1308606131426582528):
 
+
 - never have to worry about building edit/delete/undo/backup/recover type features, one generic git-backed [column] is enough
 
 - removes the need to keep additional SQL tables which keep logs of all edit histories.
@@ -99,6 +101,7 @@ This query will return:
         "message": "test_table_git_track_trigger: BEFORE UPDATE ROW on public.test_table",
         "author": "pguser ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
           {
             "field": "text",
@@ -111,6 +114,7 @@ This query will return:
         "message": "test_table_git_track_trigger: BEFORE INSERT ROW on public.test_table",
         "author": "pguser ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
           {
             "field": "id",
@@ -253,6 +257,7 @@ where identifier->>'id' = '1'
         "message": "test_table_git_track_trigger: BEFORE UPDATE ROW on public.test_table",
         "author": "pguser ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
           {
             "field": "text",
@@ -265,6 +270,7 @@ where identifier->>'id' = '1'
         "message": "test_table_git_track_trigger: BEFORE INSERT ROW on public.test_table",
         "author": "pguser ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
           {
             "field": "id",
@@ -295,7 +301,7 @@ insert into test_table(
 )
 values(
   2,
-  'a value',
+  'original value set by alice',
   '{ "commit": { "message": "some custom message", "author": { "name": "Alice", "email": "[email protected]" } } }'
 )
 ```
@@ -314,14 +320,15 @@ where id = 2
         "message": "some custom message\\n\\ntest_table_git_track_trigger: BEFORE INSERT ROW on public.test_table",
         "author": "Alice ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
           {
             "field": "id",
             "new": 2
           },
           {
             "field": "text",
-            "new": "a value"
+            "new": "original value set by alice"
           }
         ]
       }
@@ -334,13 +341,13 @@ where id = 2
 
 ```sql
 update test_table
-set text = 'a new value',
+set text = 'a new value set by admin',
     git = '{ "commit": { "message": "Changed because the previous value was out-of-date"  } }'
 where id = 2
 ```
 
 ```sql
-select git_log(git, depth := 1)
+select git_log(git, depth := 2)
 from test_table
 where id = 2
 ```
@@ -353,11 +360,28 @@ where id = 2
         "message": "Changed because the previous value was out-of-date\\n\\ntest_table_git_track_trigger: BEFORE UPDATE ROW on public.test_table",
         "author": "pguser ([email protected])",
         "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
+        "changes": [
+          {
+            "field": "text",
+            "new": "a new value set by admin",
+            "old": "original value set by alice"
+          }
+        ]
+      },
+      {
+        "message": "some custom message\\n\\ntest_table_git_track_trigger: BEFORE INSERT ROW on public.test_table",
+        "author": "Alice ([email protected])",
+        "timestamp": "2020-10-23T12:00:00.000Z",
+        "oid": "[oid]",
         "changes": [
+          {
+            "field": "id",
+            "new": 2
+          },
           {
             "field": "text",
-            "new": "a new value",
-            "old": "a value"
+            "new": "original value set by alice"
           }
         ]
       }
@@ -367,6 +391,80 @@ where id = 2
 ```
 
 By setting `depth := 1`, only the most recent change is returned.
+
+### Restoring previous versions
+
+`git_resolve` gives you a json representation of a prior version of a row, which can be used for backup and restore. The first argument is a `git` json value, the second value is a valid git ref string.
+
+Combine it with `git_log` to get a previous version - the below query uses `->1->'oid'` to get the oid from the second item in the log array:
+
+```sql
+select git_resolve(git, git_log(git)->1->>'oid')
+from test_table
+where id = 2
+```
+
+```json
+[
+  {
+    "git_resolve": {
+      "id": 2,
+      "text": "original value set by alice"
+    }
+  }
+]
+```
+
+This can be used in an update query to revert a change:
+
+```sql
+update test_table set (id, text) =
+(
+  select id, text
+  from json_populate_record(
+    null::test_table, (
+      select git_resolve(git, git_log(git)->1->>'oid')
+      from test_table
+      where id = 2
+    )
+  )
+)
+where id = 2
+returning id, text
+```
+
+```json
+[
+  {
+    "id": 2,
+    "text": "original value set by alice"
+  }
+]
+```
+
+Or a similar technique can restore a deleted item:
+
+```sql
+insert into test_table
+select * from json_populate_record(
+  null::test_table,
+  (
+    select git_resolve(git, git_log(git, depth := 1)->0->>'oid')
+    from deleted_history
+    where tablename = 'test_table' and identifier->>'id' = '1'
+  )
+)
+returning id, text
+```
+
+```json
+[
+  {
+    "id": 1,
+    "text": "updated content"
+  }
+]
+```
 <!-- codegen:end -->
 
 ## Caveat
@@ -375,6 +473,7 @@ By setting `depth := 1`, only the most recent change is returned.
 - It hasn't been performance-tested yet. It works well for rows with small, easily-json-stringifiable data. Large, frequently updated rows may hit issues.
 - It currently uses the `JSON` data type to store a serialised copy of the `.git` repo folder. This can likely be optimised to use `BYTEA` or another data type.
 - It uses several tools that were _not_ built with each other in mind (although each is well-designed and flexible enough for them to play nice without too many problems). See the [implementation section](#implementation)
+- It's still in v0, so breaking changes may occur.
 
 ## Implementation
 

scripts/generate-queries.ts

@@ -29,6 +29,13 @@ export const getQuery = (js: string) => {
     ${quotes}
     language plv8;
 
+    create or replace function git_resolve(git_json json, ref text) returns json as
+    ${quotes}
+      ${getFunctionBody(js)}
+      return module.exports.gitResolve(git_json, ref)
+    ${quotes}
+    language plv8;
+
     create or replace function git_log(git_json json, depth int) returns json as
     ${quotes}
       ${getFunctionBody(js)}

src/git.ts

@@ -5,6 +5,7 @@ import * as serializer from './serializer'
 import {SyncPromise} from './sync-promise'
 import {PG_Vars} from './pg-types'
 import {setupMemfs} from './fs'
+import {plog} from './pg-log'
 
 function writeGitFiles(gitFiles: any, fs: memfs.IFs) {
   Object.keys(gitFiles).map(filepath => {
@@ -70,6 +71,9 @@ export const rowToRepo = ({OLD, NEW, ...pg}: PG_Vars) => {
     }))
 }
 
+type TreeInfo = {type: string; content: string; oid: string}
+type WalkResult = {filepath: string; ChildInfo: TreeInfo; ParentInfo?: TreeInfo}
+
 /**
  * When passed a json object representing the `.git` folder of a repo, returns a list
  * of changes made to corresponding row. Optionally, pass `depth` to limit how far back
@@ -79,9 +83,6 @@ export const gitLog = (gitRepoJson: object, depth?: number) => {
   const {fs} = setupMemfs()
   const repo = {fs, dir: '/repo'}
 
-  type TreeInfo = {type: string; content: string; oid: string}
-  type WalkResult = {filepath: string; ChildInfo: TreeInfo; ParentInfo?: TreeInfo}
-
   return SyncPromise.resolve()
     .then(() => writeGitFiles(gitRepoJson, fs))
     .then(() => git.log({...repo, depth}))
@@ -103,6 +104,7 @@ export const gitLog = (gitRepoJson: object, depth?: number) => {
               message: e.commit.message.trim(),
               author: `${e.commit.author.name} (${e.commit.author.email})`,
               timestamp: new Date(e.commit.author.timestamp * 1000).toISOString(),
+              oid: e.oid,
               changes: results
                 .filter(
                   r => r.ChildInfo?.type === 'blob' && r.filepath !== '.' && r.ChildInfo.oid !== r.ParentInfo?.oid,
@@ -118,12 +120,42 @@ export const gitLog = (gitRepoJson: object, depth?: number) => {
     })
 }
 
+/**
+ * Resolves a git ref into a dictionary of values, which can be passed to `INSERT` or `UPDATE` operations
+ * @param gitRepoJson a json object representing the `.git` folder of a repo
+ * @param ref a git ref string
+ */
+export const gitResolve = (gitRepoJson: object, ref: string) => {
+  const {fs} = setupMemfs()
+  const repo = {fs, dir: '/repo'}
+
+  return SyncPromise.resolve()
+    .then(() => writeGitFiles(gitRepoJson, fs))
+    .then(() =>
+      git.walk({
+        ...repo,
+        trees: [git.TREE({ref})],
+        map: (filepath, entries) => resolveTree(entries![0])!.then(tree => ({filepath, tree})),
+      }),
+    )
+    .then((results: Array<{filepath: string; tree: ResolvedTree}>) =>
+      results
+        .filter(r => r.tree.type === 'blob' && r.filepath !== '.')
+        .reduce(
+          (dict, next) => Object.assign(dict, {[next.filepath]: serializer.parse(next.tree.content)}),
+          {} as Record<string, any>,
+        ),
+    )
+}
+
 /**
  * for some reason A.content() converts from a buffer to {"0": 100, "1": 101} format.
  * Object.values(...) converts back to a number array. Wasteful, but works for now.
  */
 const btos = (obj: any) => Buffer.from(Object.values<number>(obj || {})).toString()
 
+type PromiseResult<P> = P extends Promise<infer X> ? X : never
+type ResolvedTree = PromiseResult<ReturnType<typeof resolveTree>>
 /** gets the type, content and oid for a `WalkerEntry` */
 const resolveTree = (tree: git.WalkerEntry | undefined) => {
   const promises = tree && [tree.type(), tree.content().then(btos), tree.oid()]

src/index.ts

@@ -14,3 +14,4 @@ export const syncify = <A extends unknown[], R>(func: (...args: A) => Promise<R>
 
 export const rowToRepo = syncify(git.rowToRepo)
 export const gitLog = syncify(git.gitLog)
+export const gitResolve = syncify(git.gitResolve)

src/pg-log.ts

@@ -5,7 +5,7 @@ declare const plv8: any, WARNING: any
 /**
  * Wrapper for `plv8.elog` which outputs to a `docker-compose up` console.
  */
-export const plog = (...args: any[]) => {
+export const plog = (...args: any[]): undefined => {
   if (typeof plv8 === 'undefined') return
   // inspect is an easy way to pretty-print any value
   const s = inspect(args).slice(1, -1).trim()

test/result-printer.ts

@@ -9,7 +9,7 @@ export const fuzzifyDate = (s: string) => {
 
 const gitRepoHashes: string[] = []
 /**
- * JSON.stringify with a replacer that returns stable values for byte arrays, git repo json representations and timestamps.
+ * JSON.stringify with a replacer that returns stable values for byte arrays, oids, git repo json representations and timestamps.
  * Useful for jest snapshot testing - the result is pretty human readable and stays the same across runs.
  */
 export const readableJson = (o: unknown) => {
@@ -43,6 +43,9 @@ export const readableJson = (o: unknown) => {
       })
       return copy
     }
+    if (k === 'oid' && typeof v === 'string') {
+      return '[oid]'
+    }
     if (k === 'timestamp' && typeof v === 'string') {
       return fuzzifyDate(v).toISOString()
     }