Add Milestone Snapshots

This non-breaking change introduces the concept of "Milestone
Snapshots". A milestone snapshot is a snapshot of a document for a given
version, which is persisted in the database. The purpose of this is to
speed up the `Backend.fetchSnapshot` method, which currently has to
fetch all the ops required to build a snapshot from v0. Instead,
`fetchSnapshot` can now fetch the most recent, relevant milestone
snapshot and build on top of that with fewer ops.

In order to do this, the database adapter API has been updated to
include two new methods:

- `saveMilestoneSnapshot(collection, snapshot, callback): void;` stores
  the provided snapshot against the collection
- `getMilestoneSnapshot(collection, id, version, callback): void`
  fetches the most recent snapshot whose version is equal to or less
  than the provided `version` (or the most recent version if version is
  `null`, in keeping with the `to` argument in `getOps`).

The adapter also has the responsibility of saving the appropriate
milestone snapshots when a new op is committed.

Author: Alec Gibson

README.md

@@ -420,3 +420,5 @@ The `41xx` and `51xx` codes are reserved for use by ShareDB DB adapters, and the
 * 5016 - _unsubscribe PubSub method unimplemented
 * 5017 - _publish PubSub method unimplemented
 * 5018 - Required QueryEmitter listener not assigned
+* 5019 - saveMilestoneSnapshot DB method unimplemented
+* 5020 - Milestone snapshots are disabled

lib/backend.js

@@ -609,40 +609,53 @@ Backend.prototype.fetchSnapshot = function(agent, index, id, version, callback)
 };
 
 Backend.prototype._fetchSnapshot = function (collection, id, version, callback) {
-  // Bypass backend.getOps so that we don't call _sanitizeOps. We want to avoid this, because:
-  // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots
-  // - we handle the projection in _sanitizeSnapshots
-  this.db.getOps(collection, id, 0, version, null, function (error, ops) {
+  var db = this.db;
+  db.getMilestoneSnapshot(collection, id, version, function (error, milestoneSnapshot) {
     if (error) return callback(error);
 
-    var type = null;
-    var data;
-    var fetchedVersion = 0;
+    // Bypass backend.getOps so that we don't call _sanitizeOps. We want to avoid this, because:
+    // - we want to avoid the 'op' middleware, because we later use the 'readSnapshots' middleware in _sanitizeSnapshots
+    // - we handle the projection in _sanitizeSnapshots
+    var from = milestoneSnapshot ? milestoneSnapshot.v : 0;
+    db.getOps(collection, id, from, version, null, function (error, ops) {
+      if (error) return callback(error);
 
-    for (var index = 0; index < ops.length; index++) {
-      var op = ops[index];
-      fetchedVersion = op.v + 1;
+      var type = null;
+      var data;
+      var fetchedVersion = 0;
 
-      if (op.create) {
-        type = types.map[op.create.type];
+      if (milestoneSnapshot) {
+        type = types.map[milestoneSnapshot.type];
         if (!type) return callback({ code: 4008, message: 'Unknown type' });
-        data = type.create(op.create.data);
-      } else if (op.del) {
-        data = undefined;
-        type = null;
-      } else {
-        data = type.apply(data, op.op);
+        data = milestoneSnapshot.data;
+        fetchedVersion = milestoneSnapshot.v;
       }
-    }
 
-    type = type ? type.uri : null;
+      for (var index = 0; index < ops.length; index++) {
+        var op = ops[index];
+        fetchedVersion = op.v + 1;
+
+        if (op.create) {
+          type = types.map[op.create.type];
+          if (!type) return callback({ code: 4008, message: 'Unknown type' });
+          data = type.create(op.create.data);
+        } else if (op.del) {
+          data = undefined;
+          type = null;
+        } else {
+          data = type.apply(data, op.op);
+        }
+      }
 
-    if (version > fetchedVersion) {
-      return callback({ code: 4024, message: 'Requested version exceeds latest snapshot version' });
-    }
+      type = type ? type.uri : null;
 
-    var snapshot = new Snapshot(id, fetchedVersion, type, data, null);
-    callback(null, snapshot);
+      if (version > fetchedVersion) {
+        return callback({ code: 4024, message: 'Requested version exceeds latest snapshot version' });
+      }
+
+      var snapshot = new Snapshot(id, fetchedVersion, type, data, null);
+      callback(null, snapshot);
+    });
   });
 };
 

lib/db/index.js

@@ -4,6 +4,14 @@ var ShareDBError = require('../error');
 function DB(options) {
   // pollDebounce is the minimum time in ms between query polls
   this.pollDebounce = options && options.pollDebounce;
+
+  this.milestoneSnapshots = (options && options.milestoneSnapshots) || {};
+  // Whether we should store/fetch milestone snapshots
+  this.milestoneSnapshots.enabled = typeof this.milestoneSnapshots.enabled === 'boolean'
+    ? this.milestoneSnapshots.enabled
+    : false;
+  // The number of versions to skip before storing the next milestone snapshot
+  this.milestoneSnapshots.interval = this.milestoneSnapshots.interval || 1000;
 }
 module.exports = DB;
 
@@ -103,3 +111,11 @@ DB.prototype.canPollDoc = function() {
 DB.prototype.skipPoll = function() {
   return false;
 };
+
+DB.prototype.getMilestoneSnapshot = function(collection, id, version, callback) {
+  callback(null, undefined);
+};
+
+DB.prototype.saveMilestoneSnapshot = function(collection, snapshot, callback) {
+  callback(new ShareDBError(5019, 'saveMilestoneSnapshot DB method unimplemented'));
+};

lib/db/memory.js

@@ -1,5 +1,6 @@
 var DB = require('./index');
 var Snapshot = require('../snapshot');
+var ShareDBError = require('../error');
 
 // In-memory ShareDB database
 //
@@ -22,6 +23,9 @@ function MemoryDB(options) {
   // the list.
   this.ops = {};
 
+  // Map form collection name -> doc id -> array of milestone snapshots
+  this._milestoneSnapshots = {};
+
   this.closed = false;
 };
 module.exports = MemoryDB;
@@ -48,7 +52,16 @@ MemoryDB.prototype.commit = function(collection, id, op, snapshot, options, call
     if (err) return callback(err);
     err = db._writeSnapshotSync(collection, id, snapshot);
     if (err) return callback(err);
+
     var succeeded = true;
+
+    if (db._shouldSaveMilestoneSnapshot(snapshot)) {
+      return db.saveMilestoneSnapshot(collection, snapshot, function (error) {
+        if (error) return callback(error);
+        callback(null, succeeded);
+      });
+    }
+
     callback(null, succeeded);
   });
 };
@@ -171,6 +184,54 @@ MemoryDB.prototype._getVersionSync = function(collection, id) {
   return (collectionOps && collectionOps[id] && collectionOps[id].length) || 0;
 };
 
+MemoryDB.prototype.getMilestoneSnapshot = function (collection, id, version, callback) {
+  if (!this.milestoneSnapshots.enabled) {
+    return callback(null, undefined);
+  }
+
+  var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, id);
+
+  let milestoneSnapshot;
+  for (var i = 0; i < milestoneSnapshots.length; i++) {
+    var nextMilestoneSnapshot = milestoneSnapshots[i];
+    if (nextMilestoneSnapshot.v <= version || version === null) {
+      milestoneSnapshot = nextMilestoneSnapshot;
+    } else {
+      break;
+    }
+  }
+
+  callback(null, milestoneSnapshot);
+};
+
+MemoryDB.prototype.saveMilestoneSnapshot = function (collection, snapshot, callback) {
+  if (!this.milestoneSnapshots.enabled) {
+    return callback(new ShareDBError(5020, 'Milestone snapshots are disabled'));
+  }
+
+  if (!snapshot) {
+    return callback(null);
+  }
+
+  var milestoneSnapshots = this._getMilestoneSnapshotsSync(collection, snapshot.id);
+  milestoneSnapshots.push(snapshot);
+  milestoneSnapshots.sort(function (a, b) {
+    return a.v - b.v;
+  });
+
+  callback(null);
+};
+
+MemoryDB.prototype._getMilestoneSnapshotsSync = function (collection, id) {
+  var collectionSnapshots = this._milestoneSnapshots[collection] || (this._milestoneSnapshots[collection] = {});
+  return collectionSnapshots[id] || (collectionSnapshots[id] = []);
+};
+
+MemoryDB.prototype._shouldSaveMilestoneSnapshot = function (snapshot) {
+  return this.milestoneSnapshots.enabled
+    && snapshot.v % this.milestoneSnapshots.interval === 0;
+}
+
 function clone(obj) {
   return (obj === undefined) ? undefined : JSON.parse(JSON.stringify(obj));
 }

package.json

@@ -16,7 +16,8 @@
     "expect.js": "^0.3.1",
     "istanbul": "^0.4.2",
     "jshint": "^2.9.2",
-    "mocha": "^5.2.0"
+    "mocha": "^5.2.0",
+    "sinon": "^6.1.5"
   },
   "scripts": {
     "test": "./node_modules/.bin/mocha && npm run jshint",

test/client/snapshot-request.js

@@ -1,5 +1,7 @@
 var Backend = require('../../lib/backend');
 var expect = require('expect.js');
+var MemoryDB = require('../../lib/db/memory');
+var sinon = require('sinon');
 
 describe('SnapshotRequest', function () {
   var backend;
@@ -353,4 +355,53 @@ describe('SnapshotRequest', function () {
       });
     });
   });
+
+  describe('milestone snapshots enabled for every other version', function () {
+    var db;
+
+    beforeEach(function (done) {
+      var options = {
+        milestoneSnapshots: {
+          enabled: true,
+          interval: 2
+        }
+      };
+      db = new MemoryDB(options);
+      backend = new Backend({ db: db });
+
+      var tests = this;
+      db.saveMilestoneSnapshot('test-implementation', undefined, function (error) {
+        // Only run this test block if milestone snapshots are implemented on the driver
+        if (error) {
+          if (error.code === 5019) return tests.skip();
+          if (error.code !== 5020) return done(error);
+        }
+        done();
+      });
+    });
+
+    it('fetches a snapshot using the milestone', function (done) {
+      var doc = backend.connect().get('books', 'mocking-bird');
+      doc.create({ title: 'To Kill a Mocking Bird' }, function (error) {
+        if (error) return done(error);
+        doc.submitOp({ p: ['author'], oi: 'Harper Lea' }, function (error) {
+          if (error) return done(error);
+          doc.submitOp({ p: ['author'], od: 'Harper Lea', oi: 'Harper Lee' }, function (error) {
+            if (error) return done(error);
+            sinon.spy(db, 'getMilestoneSnapshot');
+            sinon.spy(db, 'getOps');
+            backend.connect().fetchSnapshot('books', 'mocking-bird', 3, function (error, snapshot) {
+              if (error) return done(error);
+              expect(db.getMilestoneSnapshot.calledOnce).to.be(true);
+              expect(db.getOps.calledWith('books', 'mocking-bird', 2, 3)).to.be(true);
+
+              expect(snapshot.v).to.be(3);
+              expect(snapshot.data).to.eql({ title: 'To Kill a Mocking Bird', author: 'Harper Lee' });
+              done();
+            });
+          });
+        });
+      });
+    });
+  });
 });

test/db-memory.js

@@ -119,8 +119,12 @@ function snapshotComparator(sortProperties) {
 
 // Run all the DB-based tests against the BasicQueryableMemoryDB.
 require('./db')({
-  create: function(callback) {
-    var db = new BasicQueryableMemoryDB();
+  create: function(options, callback) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = null;
+    }
+    var db = new BasicQueryableMemoryDB(options);
     callback(null, db);
   },
   getQuery: function(options) {