[esArchiver] combine elasticDump and ScenarioManager

package.json

@@ -201,12 +201,11 @@
     "auto-release-sinon": "1.0.3",
     "babel-eslint": "6.1.2",
     "chai": "3.5.0",
+    "chance": "1.0.6",
     "cheerio": "0.22.0",
     "chokidar": "1.6.0",
     "chromedriver": "2.24.1",
     "classnames": "2.2.5",
-    "del": "1.2.1",
-    "elasticdump": "2.1.1",
     "enzyme": "2.7.0",
     "eslint": "3.11.1",
     "eslint-plugin-babel": "4.0.0",

scripts/README.md

@@ -0,0 +1,16 @@
+# kibana dev scripts
+
+This directory contains scripts useful for interacting with Kibana tools in development. Use the node executable and `--help` flag to learn about how they work:
+
+```sh
+node scripts/{{script name}} --help
+```
+
+## for developers
+
+This directory is excluded from the build and tools within it should help users discover their capabilities. Each script in this directory must:
+
+- include the `../src/optimize/babel/register` module to bootstrap babel
+- call out to source code that is in the `src` directory
+- react to the `--help` flag
+- run everywhere OR check and fail fast when a required OS or toolchain is not available

scripts/es_archiver.js

@@ -0,0 +1,2 @@
+require('../src/optimize/babel/register');
+require('../src/es_archiver/cli');

scripts/load_dump_data.js

@@ -0,0 +1,2 @@
+require('../src/optimize/babel/register');
+require('../src/es_archiver/load_dump_data');

src/core_plugins/status_page/public/status_page.html

@@ -1,4 +1,4 @@
-<div class="container overall_state_default overall_state_{{ui.serverState}}">
+<div data-test-subj="statusPageContainer" class="container overall_state_default overall_state_{{ui.serverState}}">
   <header>
     <h1>
       Status: <span class="overall_state_color">{{ ui.serverStateMessage }}</span>

src/es_archiver/actions/index.js

@@ -0,0 +1,4 @@
+export { saveAction } from './save';
+export { loadAction } from './load';
+export { unloadAction } from './unload';
+export { rebuildAllAction } from './rebuild_all';

src/es_archiver/actions/load.js

@@ -0,0 +1,39 @@
+import { resolve } from 'path';
+import { createReadStream } from 'fs';
+
+import {
+  createPromiseFromStreams
+} from '../../utils';
+
+import {
+  isGzip,
+  createStats,
+  prioritizeMappings,
+  getArchiveFiles,
+  createParseArchiveStreams,
+  createCreateIndexStream,
+  createIndexDocRecordsStream,
+} from '../lib';
+
+export async function loadAction({ name, skipExisting, client, dataDir, log }) {
+  const inputDir = resolve(dataDir, name);
+  const stats = createStats(name, log);
+
+  const files = prioritizeMappings(await getArchiveFiles(inputDir));
+  for (const filename of files) {
+    log.info('[%s] Loading %j', name, filename);
+
+    await createPromiseFromStreams([
+      createReadStream(resolve(inputDir, filename)),
+      ...createParseArchiveStreams({ gzip: isGzip(filename) }),
+      createCreateIndexStream({ client, stats, skipExisting }),
+      createIndexDocRecordsStream(client, stats),
+    ]);
+  }
+
+  stats.forEachIndex((index, { docs }) => {
+    log.info('[%s] Indexed %d docs into %j', name, docs.indexed, index);
+  });
+
+  return stats.toJSON();
+}

src/es_archiver/actions/rebuild_all.js

@@ -0,0 +1,47 @@
+import { resolve } from 'path';
+import {
+  rename,
+  readdir,
+  createReadStream,
+  createWriteStream
+} from 'fs';
+
+import { fromNode } from 'bluebird';
+
+import {
+  createPromiseFromStreams
+} from '../../utils';
+
+import {
+  prioritizeMappings,
+  getArchiveFiles,
+  isGzip,
+  createParseArchiveStreams,
+  createFormatArchiveStreams,
+} from '../lib';
+
+export async function rebuildAllAction({ dataDir, log }) {
+  const archiveNames = await fromNode(cb => readdir(dataDir, cb));
+
+  for (const name of archiveNames) {
+    const inputDir = resolve(dataDir, name);
+    const files = prioritizeMappings(await getArchiveFiles(inputDir));
+    for (const filename of files) {
+      log.info('[%s] Rebuilding %j', name, filename);
+
+      const path = resolve(inputDir, filename);
+      const gzip = isGzip(path);
+      const tempFile = path + (gzip ? '.rebuilding.gz' : '.rebuilding');
+
+      await createPromiseFromStreams([
+        createReadStream(path),
+        ...createParseArchiveStreams({ gzip }),
+        ...createFormatArchiveStreams({ gzip }),
+        createWriteStream(tempFile),
+      ]);
+
+      await fromNode(cb => rename(tempFile, path, cb));
+      log.info('[%s] Rebuilt %j', name, filename);
+    }
+  }
+}

src/es_archiver/actions/save.js

@@ -0,0 +1,55 @@
+import { resolve } from 'path';
+import { createWriteStream } from 'fs';
+
+import { fromNode } from 'bluebird';
+import mkdirp from 'mkdirp';
+
+import {
+  createListStream,
+  createPromiseFromStreams,
+} from '../../utils';
+
+import {
+  createStats,
+  createGenerateIndexRecordsStream,
+  createFormatArchiveStreams,
+  createGenerateDocRecordsStream,
+} from '../lib';
+
+export async function saveAction({ name, indices, client, dataDir, log }) {
+  const outputDir = resolve(dataDir, name);
+  const stats = createStats(name, log);
+
+  log.info('[%s] Creating archive of %j', name, indices);
+
+  await fromNode(cb => mkdirp(outputDir, cb));
+  const resolvedIndexes = Object.keys(await client.indices.get({
+    index: indices,
+    feature: ['_settings'],
+    filterPath: ['*.settings.index.uuid']
+  }));
+
+  await Promise.all([
+    // export and save the matching indices to mappings.json
+    createPromiseFromStreams([
+      createListStream(resolvedIndexes),
+      createGenerateIndexRecordsStream(client, stats),
+      ...createFormatArchiveStreams(),
+      createWriteStream(resolve(outputDir, 'mappings.json')),
+    ]),
+
+    // export all documents from matching indexes into data.json.gz
+    createPromiseFromStreams([
+      createListStream(resolvedIndexes),
+      createGenerateDocRecordsStream(client, stats),
+      ...createFormatArchiveStreams({ gzip: true }),
+      createWriteStream(resolve(outputDir, 'data.json.gz'))
+    ])
+  ]);
+
+  stats.forEachIndex((index, { docs }) => {
+    log.info('[%s] Archived %d docs from %j', name, docs.archived, index);
+  });
+
+  return stats.toJSON();
+}

src/es_archiver/actions/unload.js

@@ -0,0 +1,35 @@
+import { resolve } from 'path';
+import { createReadStream } from 'fs';
+
+import {
+  createPromiseFromStreams
+} from '../../utils';
+
+import {
+  isGzip,
+  createStats,
+  prioritizeMappings,
+  getArchiveFiles,
+  createParseArchiveStreams,
+  createFilterRecordsStream,
+  createDeleteIndexStream
+} from '../lib';
+
+export async function unloadAction({ name, client, dataDir, log }) {
+  const inputDir = resolve(dataDir, name);
+  const stats = createStats(name, log);
+
+  const files = prioritizeMappings(await getArchiveFiles(inputDir));
+  for (const filename of files) {
+    log.info('[%s] Unloading indices from %j', name, filename);
+
+    await createPromiseFromStreams([
+      createReadStream(resolve(inputDir, filename)),
+      ...createParseArchiveStreams({ gzip: isGzip(filename) }),
+      createFilterRecordsStream('index'),
+      createDeleteIndexStream({ client, stats })
+    ]);
+  }
+
+  return stats.toJSON();
+}

src/es_archiver/cli.js

@@ -0,0 +1,94 @@
+/*************************************************************
+ *
+ *  Run `node scripts/es_archiver -- --help` for usage information
+ *
+ *************************************************************/
+
+import { resolve } from 'path';
+import { readFileSync } from 'fs';
+import { format as formatUrl } from 'url';
+
+import { Command } from 'commander';
+import elasticsearch from 'elasticsearch';
+
+import { EsArchiver } from './es_archiver';
+import { createLog } from './lib';
+
+const cmd = new Command('node scripts/es_archiver');
+
+cmd
+  .description(`CLI to manage archiving/restoring data in elasticsearch`)
+  .option('--es-url [url]', 'url for elasticsearch')
+  .option(`--dir [path]`, 'where archives are stored')
+  .option('--verbose', 'turn on verbose logging')
+  .option('--config [path]', 'path to a functional test config file to use for default values')
+  .on('--help', () => {
+    console.log(readFileSync(resolve(__dirname, './cli_help.txt'), 'utf8'));
+  });
+
+cmd.command('save <name> <indices...>')
+  .action((name, indices) => execute('save', { name, indices }));
+
+cmd.command('load <name>')
+  .action(name => execute('load', { name }));
+
+cmd.command('unload <name>')
+  .action(name => execute('unload', { name }));
+
+cmd.command('rebuild-all')
+  .action(name => execute('rebuildAll'));
+
+cmd.parse(process.argv);
+
+const missingCommand = cmd.args.every(a => !(a instanceof Command));
+if (missingCommand) {
+  execute();
+}
+
+async function execute(operation, options) {
+  try {
+    const log = createLog(cmd.verbose ? 3 : 2);
+    log.pipe(process.stdout);
+
+    // log and count all validation errors
+    let errorCount = 0;
+    const error = (msg) => {
+      errorCount++;
+      log.error(msg);
+    };
+
+    if (!operation) error('Missing or invalid command');
+    if (!cmd.esUrl) {
+      error('You must specify either --es-url or --config flags');
+    }
+    if (!cmd.dir) {
+      error('You must specify either --dir or --config flags');
+    }
+
+    // if there was a validation error display the help
+    if (errorCount) {
+      cmd.help();
+      return;
+    }
+
+    // run!
+
+    const client = new elasticsearch.Client({
+      host: cmd.esUrl,
+      log: cmd.verbose ? 'trace' : []
+    });
+
+    try {
+      const esArchiver = new EsArchiver({
+        log,
+        client,
+        dataDir: resolve(cmd.dir)
+      });
+      await esArchiver[operation](options);
+    } finally {
+      await client.close();
+    }
+  } catch (err) {
+    console.log('FATAL ERROR', err.stack);
+  }
+}

src/es_archiver/cli_help.txt

@@ -0,0 +1,15 @@
+  Examples:
+    Dump an index to disk:
+      Save all `logstash-*` indices from http://localhost:9200 to `snapshots/my_test_data` directory
+
+      WARNING: If the `my_test_data` snapshot exists it will be deleted!
+
+      $ node scripts/es_archiver save my_test_data logstash-* --dir snapshots
+
+    Load an index from disk
+      Load the `my_test_data` snapshot from the archive directory and elasticsearch instance defined
+      in the `test/functional/config.js` config file
+
+      WARNING: If the indices exist already they will be deleted!
+
+      $ node scripts/es_archiver load my_test_data --config test/functional/config.js