diff --git a/.gitignore b/.gitignore index dfa7fbb192c..e9b27fcbf3f 100755 --- a/.gitignore +++ b/.gitignore @@ -129,3 +129,6 @@ translations_github_commit_* # Localization testing SSH keys wagtail-localize-key* + +# Database copies +*.db.archive diff --git a/copy-db.js b/copy-db.js new file mode 100644 index 00000000000..c8cbd3d7dde --- /dev/null +++ b/copy-db.js @@ -0,0 +1,91 @@ +const fs = require("fs"); +const { execSync } = require(`child_process`); + +const keepDatabase = process.argv.includes(`--keep`); +const silent = { stdio: [`ignore`, `ignore`, `ignore`] }; +const PROD_APP = `foundation-mozilla-org`; +const STAGE_APP = `foundation-mofostaging-net`; +const APP = process.argv.includes(`--prod`) ? PROD_APP : STAGE_APP; + +if (APP === STAGE_APP) { + console.log( + `Running db copy for staging, run with --prod to run for production` + ); +} + +const HEROKU_OUTPUT = run(`heroku config:get DATABASE_URL -a ${APP}`); +const HEROKU_TEXT = HEROKU_OUTPUT.toString().replaceAll(`\n`, ` `); +const URL_START = HEROKU_TEXT.indexOf(`postgres://`); +const DATABASE_URL = HEROKU_TEXT.substring(URL_START).trim(); +const ROLE = DATABASE_URL.match(/postgres:\/\/([^:]+):/)[1]; +const DUMP_FILE = `${ROLE}.db.archive`; +const DB_FLAGS = `-hpostgres -Ufoundation`; + +/** + * ... + */ +function run(cmd, ignoreThrows = false, opts = {}) { + try { + return execSync(cmd, opts); + } catch (e) { + if (ignoreThrows) return e.toString(); + throw e; + } +} + +/** + * ... + */ +function docker(cmd, ignoreThrows = false) { + cmd = `docker exec foundationmozillaorg_postgres_1 ${cmd}`; + return run(cmd, ignoreThrows); +} + +console.log(`Making sure no docker containers are running...`); +run(`docker-compose down`, true, silent); +run(`docker-compose down`, true, silent); + +console.log(`Starting postgres docker image...`); +run(`docker-compose up -d postgres`, true, silent); + +console.log(`Starting backend docker image...`); +run(`docker-compose up -d backend`, true, silent); + +console.log(`Downloading ${APP} database (this may take a while)...`); +if (!fs.existsSync(DUMP_FILE)) { + docker(`pg_dump -F c ${DATABASE_URL} > ${DUMP_FILE}`); +} + +console.log(`Resetting db...`); +docker(`dropdb ${DB_FLAGS} --if-exists wagtail`); +docker(`createdb ${DB_FLAGS} wagtail`); + +console.log(`Building user roles...`); +[ROLE, `datastudio`, `datagrip-cade`].forEach((role) => + docker(`createuser ${DB_FLAGS} -s ${role}`, true) +); + +console.log(`Importing snapshot...`); +run(`docker cp ${DUMP_FILE} foundationmozillaorg_postgres_1:/`); +docker(`pg_restore ${DB_FLAGS} -dwagtail ${DUMP_FILE}`); + +console.log(`Creating admin:admin superuser account...`); +run( + [ + `docker exec foundationmozillaorg_backend_1`, + `./dockerpythonvenv/bin/python network-api/manage.py shell -c`, + `"from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com', 'admin')"`, + ].join(` `), + true, + silent +); + +console.log(`Stopping docker images...`); +run(`docker-compose down`, true, silent); + +if (!keepDatabase) { + console.log(`Running cleanup`); + fs.unlinkSync(DUMP_FILE); +} + +console.log(`All done.`); diff --git a/docs/local_development.md b/docs/local_development.md index 8ab64c7aeab..5b0394b19bd 100644 --- a/docs/local_development.md +++ b/docs/local_development.md @@ -31,6 +31,8 @@ To get a list of invoke commands available, run `invoke -l`: manage (docker-manage) Shorthand to manage.py. inv docker-manage "[COMMAND] [ARG]" migrate (docker-migrate) Updates database schema new-db (docker-new-db) Delete your database and create a new one with fake data + copy-stage-db Overwrite your local docker postgres DB with a copy of the staging database + copy-prod-db Overwrite your local docker postgres DB with a copy of the production database new-env (docker-new-env) Get a new dev environment and a new database with fake data npm (docker-npm) Shorthand to npm. inv docker-npm "[COMMAND] [ARG]" npm-install (docker-npm-install) Install Node dependencies @@ -111,20 +113,23 @@ Use `invoke npm update`. Requirements: -- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) - Heroku Account with membership on the Mozilla team (ask in #mofo-engineering on Slack) +- [Heroku CLI](https://devcenter.heroku.com/articles/heroku-cli) - remember to run `heroku login` after installation finishes. -Some development work requires testing changes against "whatever the current production database looks like", which requires having postgresql installed locally (`brew install postgresql` on mac; download and run the official installer for windows; if you use linux/unix, you know how to install things for your favourite flavour, so just do that for postgresql). We backport prod data to staging every week, scrubbing PII, so we'll be creating a copy of that for local testing, too. +You can copy the staging database using `inv copy-stage-db`, or the production database using `inv copy-prod-db`. -**Note**: your postgres version must be compatible with the version that is used on heroku in order for the `pg_dump` command to work. In general, this means that the result of `psql --version` must be **greater or equal to** the version found when running `heroku pg:info -a foundation-mofostaging-net` (look for "PG Version") +**Note** that this script requires that docker is not already ready running, as this script requires exclusive database access. You can ensure that this is the case by running `docker-compose down` twice in the repo's root directory. The first time should show all running containers getting shut down, the second should confirm that there is nothing to take down anymore. -The steps involved in cloning the database for local use are as follows: +For more control, you can also manually invoke `node copy-db.js`, which has the following behavior: -1. Run `docker-compose up postgres` to start the `postgres` service without starting the rest of the server setup (note that if you want to detach stdout, add the `-d` flag to the command) -2. Drop the existing `wagtail` database in the PostgreSQL server inside your docker environment with `dropdb --if-exists -h localhost -p 5678 -U foundation wagtail` -3. Use the Heroku CLI to pull the remote database into your local docker PostgreSQL server with `heroku pg:pull -a foundation-mofostaging-net DATABASE_URL postgresql://foundation@localhost:5678/wagtail` +``` + node copy-db.js Copy the staging database. + node copy-db.js --prod Copy the production database. +``` + +In addition, you can add a `--keep` runtime flag when invoking the script, in which case the database dump file will not be deleted after completion. -If you need to reset this database, running through these steps again will get you back into sync with staging. +If the copy script is invoked when the correct database dump file already exists, it will not redownload it and simply reuse the file on disk. --- diff --git a/tasks.py b/tasks.py index 40e7d3f527c..23065a05203 100644 --- a/tasks.py +++ b/tasks.py @@ -186,6 +186,18 @@ def npm_install(ctx): ctx.run("docker-compose run --rm watch-static-files npm install") +@task(aliases=["copy-stage-db"]) +def copy_staging_database(ctx): + with ctx.cd(ROOT): + ctx.run("node copy-db.js") + + +@task(aliases=["copy-prod-db"]) +def copy_production_database(ctx): + with ctx.cd(ROOT): + ctx.run("node copy-db.js --prod") + + # Django shorthands @task(aliases=["docker-manage"]) def manage(ctx, command):