Import/export support for emulators

scripts/triggers-end-to-end-tests/.gitignore

@@ -9,6 +9,12 @@ database-debug.log*
 firestore-debug.log*
 pubsub-debug.log*
 
+# NPM
+package-lock.json
+
+# Test data
+data/
+
 # Firebase cache
 .firebase/
 
@@ -65,4 +71,4 @@ node_modules/
 .yarn-integrity
 
 # dotenv environment variables file
-.env
+.env

scripts/triggers-end-to-end-tests/firebase.json

@@ -5,6 +5,9 @@
   },
   "functions": {},
   "emulators": {
+    "hub": {
+      "port": 4000
+    },
     "database": {
       "port": 9000
     },

scripts/triggers-end-to-end-tests/run.spec.js

@@ -32,7 +32,6 @@ const ALL_EMULATORS_STARTED_LOG = "All emulators started, it is now safe to conn
  * parallel emulator subprocesses.
  */
 const TEST_SETUP_TIMEOUT = 60000;
-const EMULATORS_STARTUP_DELAY_TIMEOUT = 60000;
 const EMULATORS_WRITE_DELAY_MS = 5000;
 const EMULATORS_SHUTDOWN_DELAY_MS = 5000;
 const EMULATOR_TEST_TIMEOUT = EMULATORS_WRITE_DELAY_MS * 2;
@@ -44,6 +43,66 @@ const EMULATOR_TEST_TIMEOUT = EMULATORS_WRITE_DELAY_MS * 2;
 const FIRESTORE_COMPLETION_MARKER = "test/done_from_firestore";
 const DATABASE_COMPLETION_MARKER = "test/done_from_database";
 
+function CLIProcess(name) {
+  this.name = name;
+  this.process = undefined;
+}
+CLIProcess.prototype.constructor = CLIProcess;
+
+CLIProcess.prototype.start = function(cmd, additionalArgs, logDoneFn) {
+  const args = [PROJECT_ROOT + "/lib/bin/firebase.js", cmd, "--project", FIREBASE_PROJECT];
+
+  if (additionalArgs) {
+    args.push(...additionalArgs);
+  }
+
+  this.process = subprocess.spawn("node", args);
+
+  this.process.stdout.on("data", (data) => {
+    process.stdout.write(`[${this.name} stdout] ` + data);
+  });
+
+  this.process.stderr.on("data", (data) => {
+    console.log(`[${this.name} stderr] ` + data);
+  });
+
+  let started;
+  if (logDoneFn) {
+    started = new Promise((resolve) => {
+      this.process.stdout.on("data", (data) => {
+        if (logDoneFn(data)) {
+          resolve();
+        }
+      });
+    });
+  } else {
+    started = new Promise((resolve) => {
+      this.process.once("close", () => {
+        this.process = undefined;
+        resolve();
+      });
+    });
+  }
+
+  return started;
+};
+
+CLIProcess.prototype.stop = function() {
+  if (!this.process) {
+    return Promise.resolve();
+  }
+
+  const stopped = new Promise((resolve) => {
+    this.process.once("close", (/* exitCode, signal */) => {
+      this.process = undefined;
+      resolve();
+    });
+  });
+
+  this.process.kill("SIGINT");
+  return stopped;
+};
+
 function TriggerEndToEndTest(config) {
   this.rtdb_emulator_host = "localhost";
   this.rtdb_emulator_port = config.emulators.database.port;
@@ -69,7 +128,7 @@ function TriggerEndToEndTest(config) {
   this.rtdb_from_rtdb = false;
   this.firestore_from_firestore = false;
 
-  this.emulators_process = null;
+  this.cli_process = null;
 }
 
 /*
@@ -86,22 +145,13 @@ TriggerEndToEndTest.prototype.success = function success() {
 };
 
 TriggerEndToEndTest.prototype.startEmulators = function startEmulators(additionalArgs) {
-  var self = this;
-  const args = [
-    PROJECT_ROOT + "/lib/bin/firebase.js",
-    "emulators:start",
-    "--project",
-    FIREBASE_PROJECT,
-  ];
-
-  if (additionalArgs) {
-    args.push(...additionalArgs);
-  }
-
-  self.emulators_process = subprocess.spawn("node", args);
+  const self = this;
+  const cli = new CLIProcess("default");
+  const started = cli.start("emulators:start", additionalArgs, (data) => {
+    return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+  });
 
-  self.emulators_process.stdout.on("data", function(data) {
-    process.stdout.write("[emulators stdout] " + data);
+  cli.process.stdout.on("data", function(data) {
     if (data.indexOf(RTDB_FUNCTION_LOG) > -1) {
       self.rtdb_trigger_count++;
     }
@@ -111,25 +161,21 @@ TriggerEndToEndTest.prototype.startEmulators = function startEmulators(additiona
     if (data.indexOf(PUBSUB_FUNCTION_LOG) > -1) {
       self.pubsub_trigger_count++;
     }
-    if (data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1) {
-      self.all_emulators_started = true;
-    }
   });
 
-  self.emulators_process.stderr.on("data", function(data) {
-    console.log("[emulators stderr] " + data);
-  });
+  this.cli_process = cli;
+  return started;
 };
 
-TriggerEndToEndTest.prototype.stopEmulators = function stopEmulators(done) {
-  this.emulators_process.once("close", function(/* exitCode, signal */) {
-    done();
-  });
+TriggerEndToEndTest.prototype.startEmulatorsAndWait = function startEmulatorsAndWait(
+  additionalArgs,
+  done
+) {
+  this.startEmulators(additionalArgs).then(done);
+};
 
-  /*
-   * CLI process only shuts down emulators cleanly on SIGINT.
-   */
-  this.emulators_process.kill("SIGINT");
+TriggerEndToEndTest.prototype.stopEmulators = function stopEmulators(done) {
+  this.cli_process.stop().then(done);
 };
 
 TriggerEndToEndTest.prototype.invokeHttpFunction = function invokeHttpFunction(name, done) {
@@ -217,15 +263,7 @@ describe("database and firestore emulator function triggers", function() {
           });
         },
         function(done) {
-          test.startEmulators(["--only", "functions,database,firestore"]);
-          test.waitForCondition(
-            () => test.all_emulators_started,
-            EMULATORS_STARTUP_DELAY_TIMEOUT,
-            (err) => {
-              expect(err).to.be.undefined;
-              done();
-            }
-          );
+          test.startEmulatorsAndWait(["--only", "functions,database,firestore"], done);
         },
         function(done) {
           test.firestore_client = new Firestore({
@@ -390,15 +428,7 @@ describe("pubsub emulator function triggers", function() {
           });
         },
         function(done) {
-          test.startEmulators(["--only", "functions,pubsub"]);
-          test.waitForCondition(
-            () => test.all_emulators_started,
-            EMULATORS_STARTUP_DELAY_TIMEOUT,
-            (err) => {
-              expect(err).to.be.undefined;
-              done();
-            }
-          );
+          test.startEmulatorsAndWait(["--only", "functions,pubsub"], done);
         },
       ],
       done
@@ -429,3 +459,31 @@ describe("pubsub emulator function triggers", function() {
     done();
   });
 });
+
+describe("import/export end to end", () => {
+  it("should be able to import/export firestore data", async () => {
+    // Start up emulator suite
+    const emulatorsCLI = new CLIProcess("1");
+    await emulatorsCLI.start("emulators:start", ["--only", "firestore"], (data) => {
+      return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+    });
+
+    // Ask for export
+    const exportCLI = new CLIProcess("2");
+    await exportCLI.start("emulators:export", ["./data"]);
+
+    // Stop the suite
+    await emulatorsCLI.stop();
+
+    // Attempt to import
+    await emulatorsCLI.start(
+      "emulators:start",
+      ["--only", "firestore", "--import", "data"],
+      (data) => {
+        return data.indexOf(ALL_EMULATORS_STARTED_LOG) > -1;
+      }
+    );
+
+    return emulatorsCLI.stop();
+  }).timeout(TEST_SETUP_TIMEOUT);
+});

src/commands/emulators-exec.ts

@@ -84,6 +84,7 @@ module.exports = new Command("emulators:exec <script>")
   )
   .option(commandUtils.FLAG_ONLY, commandUtils.DESC_ONLY)
   .option(commandUtils.FLAG_INSPECT_FUNCTIONS, commandUtils.DESC_INSPECT_FUNCTIONS)
+  .option(commandUtils.FLAG_IMPORT, commandUtils.DESC_IMPORT)
   .action(async (script: string, options: any) => {
     const projectId = getProjectId(options, true);
     const extraEnv: Record<string, string> = {};

src/commands/emulators-export.ts

@@ -0,0 +1,69 @@
+import * as clc from "cli-color";
+import * as fs from "fs";
+import * as path from "path";
+
+import * as api from "../api";
+import { Command } from "../command";
+import * as commandUtils from "../emulator/commandUtils";
+import * as utils from "../utils";
+import { EmulatorHub } from "../emulator/hub";
+import { FirebaseError } from "../error";
+
+module.exports = new Command("emulators:export <path>")
+  .description("export data from running emulators")
+  .option(commandUtils.FLAG_ONLY, commandUtils.DESC_ONLY)
+  .action(async (exportPath: string, options: any) => {
+    const projectId = options.project;
+    const locator = EmulatorHub.readLocatorFile(projectId);
+    if (!locator) {
+      throw new FirebaseError(
+        `Did not find any running emulators for project ${clc.bold(projectId)}.`,
+        { exit: 1 }
+      );
+    }
+    const hubOrigin = `http://${locator.host}:${locator.port}`;
+
+    try {
+      await api.request("GET", "/", {
+        origin: hubOrigin,
+      });
+    } catch (e) {
+      throw new FirebaseError(
+        `The emulator hub at ${hubOrigin} did not respond to a status check. If this error continues try shutting down all running emulators and deleting the file ${EmulatorHub.getLocatorFilePath(
+          projectId
+        )}`,
+        { exit: 1 }
+      );
+    }
+
+    utils.logBullet(
+      `Found running emulator hub for project ${clc.bold(projectId)} at ${hubOrigin}`
+    );
+
+    // If the export target directory does not exist, we should attempt to create it
+    const absPath = path.resolve(exportPath);
+    if (!fs.existsSync(absPath)) {
+      utils.logBullet(`Creating export directory ${absPath}`);
+      fs.mkdirSync(absPath);
+    }
+
+    const exportBody = {
+      path: absPath,
+    };
+
+    utils.logBullet(`Exporting data to: ${absPath}`);
+    await api
+      .request("POST", EmulatorHub.PATH_EXPORT, {
+        origin: hubOrigin,
+        json: true,
+        data: exportBody,
+      })
+      .catch((e) => {
+        throw new FirebaseError("Export request failed, see emulator logs for more information.", {
+          exit: 1,
+          original: e,
+        });
+      });
+
+    utils.logSuccess("Export complete");
+  });

src/commands/emulators-start.ts

@@ -8,6 +8,7 @@ module.exports = new Command("emulators:start")
   .description("start the local Firebase emulators")
   .option(commandUtils.FLAG_ONLY, commandUtils.DESC_ONLY)
   .option(commandUtils.FLAG_INSPECT_FUNCTIONS, commandUtils.DESC_INSPECT_FUNCTIONS)
+  .option(commandUtils.FLAG_IMPORT, commandUtils.DESC_IMPORT)
   .action(async (options: any) => {
     try {
       await controller.startAll(options);

src/commands/index.js

@@ -46,6 +46,7 @@ module.exports = function(client) {
   client.deploy = loadCommand("deploy");
   client.emulators = {};
   client.emulators.exec = loadCommand("emulators-exec");
+  client.emulators.export = loadCommand("emulators-export");
   client.emulators.start = loadCommand("emulators-start");
   client.experimental = {};
   client.experimental.functions = {};

src/emulator/commandUtils.ts

@@ -5,20 +5,23 @@ import * as utils from "../utils";
 import * as logger from "../logger";
 import requireAuth = require("../requireAuth");
 import requireConfig = require("../requireConfig");
-import { Emulators } from "../emulator/types";
+import { Emulators, ALL_SERVICE_EMULATORS } from "../emulator/types";
 import { FirebaseError } from "../error";
 
-export const FLAG_ONLY: string = "--only <emulators>";
-export const DESC_ONLY: string =
-  "only run specific emulators. " +
-  "This is a comma separated list of emulators to start. " +
+export const FLAG_ONLY = "--only <emulators>";
+export const DESC_ONLY =
+  "only specific emulators. " +
+  "This is a comma separated list of emulator names. " +
   "Valid options are: " +
-  JSON.stringify(controller.VALID_EMULATOR_STRINGS);
+  JSON.stringify(ALL_SERVICE_EMULATORS);
 
 export const FLAG_INSPECT_FUNCTIONS = "--inspect-functions [port]";
 export const DESC_INSPECT_FUNCTIONS =
   "emulate Cloud Functions in debug mode with the node inspector on the given port (9229 if not specified)";
 
+export const FLAG_IMPORT = "--import [dir]";
+export const DESC_IMPORT = "import emulator data from a previous export (see emulators:export)";
+
 /**
  * We want to be able to run the Firestore and Database emulators even in the absence
  * of firebase.json. For Functions and Hosting we require the JSON file since the

src/emulator/constants.ts

@@ -3,6 +3,7 @@ import * as url from "url";
 import { Address, Emulators } from "./types";
 
 const DEFAULT_PORTS: { [s in Emulators]: number } = {
+  hub: 4000,
   hosting: 5000,
   functions: 5001,
   firestore: 8080,