Merge pull request #362 from nhsx/master

Move in migrations from master to main

reader/babel.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  presets: [['@babel/preset-env', { targets: { node: 'current' } }]],
+};

reader/bin/migration.js

@@ -0,0 +1,43 @@
+#!/usr/bin/env node
+import fs from 'fs';
+import path from 'path';
+import { program, Option } from 'commander';
+import { up, down, make } from '../lib/migrations';
+
+program
+  .name('migration tool')
+  .description('CLI to migrate and rollback CKAN data migrations')
+  .version('0.0.1');
+
+program
+  .command('make')
+  .description(
+    'Create a new timestamped migration file and corresponding tests'
+  )
+  .argument('<name> [string]', 'name of file')
+  .action(make);
+
+program
+  .command('up')
+  .description('Run any new migrations since last run')
+  .option('--dry-run', 'Preview the migration results before persisting')
+  .option('--force', 'Force rerun of migration')
+  .addOption(
+    new Option('-e, --env <string>', 'environment')
+      .choices(['local', 'dev', 'test', 'prod'])
+      .default('local')
+  )
+  .action(up);
+
+program
+  .command('down')
+  .description('Roll back the last migration')
+  .option('--dry-run [boolean]')
+  .addOption(
+    new Option('-e, --env <string>', 'environment')
+      .choices(['local', 'dev', 'test', 'prod'])
+      .default('local')
+  )
+  .action(down);
+
+program.parse();

reader/jest.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  transformIgnorePatterns: ['node_modules/(!axios)'],
+  testPathIgnorePatterns: ['/node_modules/', '/__fixtures__/'],
+};

reader/lib/merge-record/__tests__/index.test.js

@@ -1,4 +1,4 @@
-import { fromCkan, fromParsed } from './fixture';
+import { fromCkan, fromParsed } from './__fixtures__';
 import { mergeRecord } from '../index';
 
 describe('mergeRecord', () => {

reader/lib/migrations/README.md

@@ -0,0 +1,64 @@
+# Data Migrations ✍️
+
+NB: This is a work in progress, there are probably errors
+
+## Set up environment
+
+Create API tokens for each environment you want to run the migrations against
+
+```
+LOCAL_CKAN_API_TOKEN=<LOCAL_API_TOKEN>
+LOCAL_CKAN_URL=http://localhost:5005/api/action
+
+DEV_CKAN_API_TOKEN=<DEV_API_TOKEN>
+DEV_CKAN_URL=https://manage.dev.standards.nhs.uk/api/action
+
+TEST_CKAN_API_TOKEN=<TEST_API_TOKEN>
+TEST_CKAN_URL=https://manage.test.standards.nhs.uk/api/action
+
+PROD_CKAN_API_TOKEN=<PROD_API_TOKEN>
+PROD_CKAN_URL=https://manage.standards.nhs.uk/api/action
+```
+
+## Create a new migration
+
+Create a new migration by running the following command in the root of the reader directory:
+
+```bash
+$ npm run migration make <name>
+```
+
+This will create a new migration template in `./lib`, and corresponding test template in `./__tests__`
+
+Migrations require a named export: `up`, and an optional export: `down`. If `down` is omitted the last migration is rolled back by applying the diff saved in cache.json.
+
+## Running a migration
+
+Run any new migrations since the last run.
+
+```bash
+$ npm run migration up
+```
+
+### Args
+
+* `--dry-run` - Show the affected number of rows without running the migration
+* `--env -e [local | dev | test | prod] (default local)` - The env to run against
+* `--force` - Force rerun of a run migration (WIP - need to specify which)
+
+The output of the migration is saved in `cache.json` - this should be committed to source control, however local cache should not. **TODO - split local cache to new gitignored file**
+
+## Rolling back a migration
+
+Roll back the last run migration. This will use `down` if specified, or will attempt to patch using the stored diff for each row if not.
+
+```bash
+$ npm run migration down
+```
+
+### Args
+
+* `--dry-run` - Show the affected number of rows without running the migration
+* `--env -e [local | dev | test | prod] (default local)` - The env to run against
+
+The cache.json file is updated with the result of the rolled back migration

reader/lib/migrations/__tests__/1665410686156-update-status.test.js

@@ -0,0 +1,151 @@
+import { up } from '../lib/1665410686156-update-status';
+import { fromCkan } from './__fixtures__';
+
+describe('update status', () => {
+  test('Maps in-development statuses', () => {
+    expect(up({ ...fromCkan, status: 'draft' })).toEqual({
+      ...fromCkan,
+      status: 'in-development',
+    });
+    expect(up({ ...fromCkan, status: 'in_development' })).toEqual({
+      ...fromCkan,
+      status: 'in-development',
+    });
+    expect(up({ ...fromCkan, status: 'in-development' })).toEqual({
+      ...fromCkan,
+      status: 'in-development',
+    });
+    expect(up({ ...fromCkan, status: 'In Development' })).toEqual({
+      ...fromCkan,
+      status: 'in-development',
+    });
+    expect(up({ ...fromCkan, status: 'In development' })).toEqual({
+      ...fromCkan,
+      status: 'in-development',
+    });
+  });
+
+  test('Maps active statuses', () => {
+    expect(up({ ...fromCkan, status: 'active' })).toEqual({
+      ...fromCkan,
+      status: 'active',
+    });
+    expect(up({ ...fromCkan, status: 'Active' })).toEqual({
+      ...fromCkan,
+      status: 'active',
+    });
+  });
+
+  test('Maps deprecated statuses', () => {
+    expect(up({ ...fromCkan, status: 'deprecated' })).toEqual({
+      ...fromCkan,
+      status: 'deprecated',
+    });
+    expect(up({ ...fromCkan, status: 'Deprecated' })).toEqual({
+      ...fromCkan,
+      status: 'deprecated',
+    });
+  });
+
+  test('Maps retired statuses', () => {
+    expect(up({ ...fromCkan, status: 'retired' })).toEqual({
+      ...fromCkan,
+      status: 'retired',
+    });
+    expect(up({ ...fromCkan, status: 'Retired' })).toEqual({
+      ...fromCkan,
+      status: 'retired',
+    });
+  });
+
+  test('Maps draft-in-progress statuses', () => {
+    expect(up({ ...fromCkan, status: 'draft-in-progress' })).toEqual({
+      ...fromCkan,
+      status: 'draft-in-progress',
+    });
+    expect(up({ ...fromCkan, status: 'draft_in_progress' })).toEqual({
+      ...fromCkan,
+      status: 'draft-in-progress',
+    });
+    expect(up({ ...fromCkan, status: 'Draft in progress' })).toEqual({
+      ...fromCkan,
+      status: 'draft-in-progress',
+    });
+    expect(up({ ...fromCkan, status: 'Draft in Progress' })).toEqual({
+      ...fromCkan,
+      status: 'draft-in-progress',
+    });
+  });
+
+  test('Maps awaiting-approval statuses', () => {
+    expect(up({ ...fromCkan, status: 'awaiting' })).toEqual({
+      ...fromCkan,
+      status: 'awaiting-approval',
+    });
+    expect(up({ ...fromCkan, status: 'awaiting-approval' })).toEqual({
+      ...fromCkan,
+      status: 'awaiting-approval',
+    });
+    expect(up({ ...fromCkan, status: 'awaiting_approval' })).toEqual({
+      ...fromCkan,
+      status: 'awaiting-approval',
+    });
+    expect(up({ ...fromCkan, status: 'Awaiting approval' })).toEqual({
+      ...fromCkan,
+      status: 'awaiting-approval',
+    });
+    expect(up({ ...fromCkan, status: 'Awaiting Approval' })).toEqual({
+      ...fromCkan,
+      status: 'awaiting-approval',
+    });
+  });
+
+  test('Maps proposed statuses', () => {
+    expect(up({ ...fromCkan, status: 'proposed' })).toEqual({
+      ...fromCkan,
+      status: 'proposed',
+    });
+    expect(up({ ...fromCkan, status: 'Proposed' })).toEqual({
+      ...fromCkan,
+      status: 'proposed',
+    });
+  });
+
+  test('Maps on-hold statuses', () => {
+    expect(up({ ...fromCkan, status: 'on-hold' })).toEqual({
+      ...fromCkan,
+      status: 'on-hold',
+    });
+    expect(up({ ...fromCkan, status: 'on_hold' })).toEqual({
+      ...fromCkan,
+      status: 'on-hold',
+    });
+    expect(up({ ...fromCkan, status: 'On hold' })).toEqual({
+      ...fromCkan,
+      status: 'on-hold',
+    });
+    expect(up({ ...fromCkan, status: 'On Hold' })).toEqual({
+      ...fromCkan,
+      status: 'on-hold',
+    });
+  });
+
+  test('Maps cancelled statuses', () => {
+    expect(up({ ...fromCkan, status: 'cancelled' })).toEqual({
+      ...fromCkan,
+      status: 'cancelled',
+    });
+    expect(up({ ...fromCkan, status: 'discontinued' })).toEqual({
+      ...fromCkan,
+      status: 'cancelled',
+    });
+    expect(up({ ...fromCkan, status: 'Discontinued' })).toEqual({
+      ...fromCkan,
+      status: 'cancelled',
+    });
+    expect(up({ ...fromCkan, status: 'Cancelled' })).toEqual({
+      ...fromCkan,
+      status: 'cancelled',
+    });
+  });
+});

reader/lib/migrations/__tests__/__fixtures__/index.js

@@ -0,0 +1,72 @@
+export const fromCkan = {
+  help: 'http://localhost:5005/api/3/action/help_show?name=package_show',
+  success: true,
+  result: {
+    assurance: '',
+    author: null,
+    author_email: null,
+    care_setting: [
+      'Community health',
+      'Dentistry',
+      'Hospital',
+      'Maternity',
+      'Mental health',
+      'Pharmacy',
+      'GP / Primary care',
+      'Social care',
+      'Urgent and Emergency Care',
+    ],
+    contact_details: '[email protected]',
+    creator_user_id: '888e4da0-f614-436c-a0eb-fc83fc16033c',
+    dependencies: 'Shared Care Record or other infrastructure.',
+    description:
+      'Part of a suite of social care standards, About Me information is the most important details that a person wants to share with professionals in health and social care.',
+    documentation_help_text: '',
+    documentation_link: 'https://theprsb.org/current-standards/aboutme/',
+    endorsements: '',
+    id: '392dabed-9b86-4608-80df-4cb2cd6b7d79',
+    isopen: false,
+    license_id: null,
+    license_title: null,
+    maintainer: null,
+    maintainer_email: null,
+    mandated: '\\true',
+    metadata_created: '2021-12-17T15:40:03.882353',
+    metadata_modified: '2022-04-07T11:35:08.532645',
+    name: 'about-me',
+    notes: null,
+    num_resources: 0,
+    num_tags: 0,
+    organization: {
+      id: '9d919972-3272-441f-9c6d-ed2745acb2d5',
+      name: 'professional-record-standards-body',
+      title: 'Professional Record Standards Body',
+      type: 'organization',
+      description: '',
+      image_url: '',
+      created: '2021-12-21T15:38:56.264481',
+      is_organization: true,
+      approval_status: 'approved',
+      state: 'active',
+    },
+    owner_org: '9d919972-3272-441f-9c6d-ed2745acb2d5',
+    private: false,
+    reference_code: '',
+    related_standards:
+      'Core Information Standard (CIS) or other standards that use it. E.g. the Digital Care and Support Plan and Urgent Referral from Care Home to Hospital\r\n' +
+      'Standard.',
+    standard_category: 'Record standard',
+    state: 'active',
+    status: 'active',
+    title: 'About Me',
+    topic: [],
+    type: 'dataset',
+    url: null,
+    version: null,
+    resources: [],
+    tags: [],
+    groups: [],
+    relationships_as_subject: [],
+    relationships_as_object: [],
+  },
+};