-
Notifications
You must be signed in to change notification settings - Fork 15
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Orion db migrations and state export fixes (#298)
* read/write export.json file using big-json package * patch @subsquid/typeorm-config & @subsquid/typeorm-migration packages to change 'squid-typeorm-migration apply' commad to apply a single migrations file too * regenerate the postgres db migrations * update package version and add changelog * added custom migration to set orionLanguage to all of the processed videos * update *-Data.js migration file * rename *-Operator.js migrations file * rename *-Indexes.js migrations file * patch @subsquid/openreader and @subsquid/typeorm-codegen dependencies include the db schema too in the generated postgres migrations, and a 'schema' directive to specify the schema of any entity * create *-Admin.js migration to create an admin schema & user * separate the view definitions from views migration file * create JS script to create new views migrations file. * add @Schema direcitve to hidden entities in graphql schema definitions * regenerate db/migrations * update 'generate-migrations' makefile command * updated documentation for upgrading orion and entity visibility * update CHANGELOG * create VIEWs for hidden entities too * fix: use snake case property names in createQueryBuilder instance methods
- Loading branch information
1 parent
61c0909
commit 52333e9
Showing
33 changed files
with
1,259 additions
and
719 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| diff --git a/node_modules/@subsquid/typeorm-config/lib/config.js b/node_modules/@subsquid/typeorm-config/lib/config.js | ||
| index 046611f..28fd4e2 100644 | ||
| --- a/node_modules/@subsquid/typeorm-config/lib/config.js | ||
| +++ b/node_modules/@subsquid/typeorm-config/lib/config.js | ||
| @@ -28,22 +28,22 @@ const path = __importStar(require("path")); | ||
| const process = __importStar(require("process")); | ||
| const connectionOptions_1 = require("./connectionOptions"); | ||
| const namingStrategy_1 = require("./namingStrategy"); | ||
| -exports.MIGRATIONS_DIR = 'db/migrations'; | ||
| +exports.MIGRATIONS_DIR = "db/migrations"; | ||
| function createOrmConfig(options) { | ||
| let dir = path.resolve(options?.projectDir || process.cwd()); | ||
| - let model = resolveModel(path.join(dir, 'lib/model')); | ||
| + let model = resolveModel(path.join(dir, "lib/model")); | ||
| let migrationsDir = path.join(dir, exports.MIGRATIONS_DIR); | ||
| return { | ||
| - type: 'postgres', | ||
| + type: "postgres", | ||
| namingStrategy: new namingStrategy_1.SnakeNamingStrategy(), | ||
| entities: [model], | ||
| - migrations: [migrationsDir + '/*.js'], | ||
| - ...(0, connectionOptions_1.createConnectionOptions)() | ||
| + migrations: [migrationsDir + `/${options?.file || "*.js"}`], | ||
| + ...(0, connectionOptions_1.createConnectionOptions)(), | ||
| }; | ||
| } | ||
| exports.createOrmConfig = createOrmConfig; | ||
| function resolveModel(model) { | ||
| - model = path.resolve(model || 'lib/model'); | ||
| + model = path.resolve(model || "lib/model"); | ||
| try { | ||
| return require.resolve(model); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| diff --git a/node_modules/@subsquid/typeorm-migration/lib/apply.js b/node_modules/@subsquid/typeorm-migration/lib/apply.js | ||
| index a8dd9c8..999ed2a 100644 | ||
| --- a/node_modules/@subsquid/typeorm-migration/lib/apply.js | ||
| +++ b/node_modules/@subsquid/typeorm-migration/lib/apply.js | ||
| @@ -29,10 +29,14 @@ const commander_1 = require("commander"); | ||
| const dotenv = __importStar(require("dotenv")); | ||
| const typeorm_1 = require("typeorm"); | ||
| (0, util_internal_1.runProgram)(async () => { | ||
| - commander_1.program.description('Apply pending migrations').parse(); | ||
| + commander_1.program | ||
| + .description("Apply pending migrations") | ||
| + .option("-f, --filename <type>", "Specify a filename") | ||
| + .parse(); | ||
| + const options = commander_1.program.opts(); | ||
| dotenv.config(); | ||
| let connection = new typeorm_1.DataSource({ | ||
| - ...(0, typeorm_config_1.createOrmConfig)(), | ||
| + ...(0, typeorm_config_1.createOrmConfig)({ file: options.filename }), | ||
| subscribers: [], | ||
| synchronize: false, | ||
| migrationsRun: false, | ||
| @@ -41,7 +45,7 @@ const typeorm_1 = require("typeorm"); | ||
| }); | ||
| await connection.initialize(); | ||
| try { | ||
| - await connection.runMigrations({ transaction: 'all' }); | ||
| + await connection.runMigrations({ transaction: "all" }); | ||
| } | ||
| finally { | ||
| await connection.destroy().catch(() => null); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| const fs = require('fs') | ||
| const path = require('path') | ||
|
|
||
| const migrationsDir = path.join(__dirname, 'migrations') | ||
|
|
||
| // Function to generate the new views migration file | ||
| const generateViewsMigration = (versionNumber) => { | ||
| const className = `Views${versionNumber}` | ||
| const fileName = `${versionNumber}-Views.js` | ||
| const fileContent = ` | ||
| const { getViewDefinitions } = require('../viewDefinitions') | ||
| module.exports = class ${className} { | ||
| name = '${className}' | ||
| async up(db) { | ||
| const viewDefinitions = getViewDefinitions(db); | ||
| for (const [tableName, viewConditions] of Object.entries(viewDefinitions)) { | ||
| if (Array.isArray(viewConditions)) { | ||
| await db.query(\` | ||
| CREATE OR REPLACE VIEW "\${tableName}" AS | ||
| SELECT * | ||
| FROM "admin"."\${tableName}" AS "this" | ||
| WHERE \${viewConditions.map(cond => \`(\${cond})\`).join(' AND ')} | ||
| \`); | ||
| } else { | ||
| await db.query(\` | ||
| CREATE OR REPLACE VIEW "\${tableName}" AS (\${viewConditions}) | ||
| \`); | ||
| } | ||
| } | ||
| } | ||
| async down(db) { | ||
| const viewDefinitions = this.getViewDefinitions(db) | ||
| for (const viewName of Object.keys(viewDefinitions)) { | ||
| await db.query(\`DROP VIEW "\${viewName}"\`) | ||
| } | ||
| } | ||
| } | ||
| ` | ||
|
|
||
| const filePath = path.join(migrationsDir, fileName) | ||
| fs.writeFileSync(filePath, fileContent) | ||
| console.log(`Generated new views migration: ${filePath}`) | ||
| } | ||
|
|
||
| // Get the latest data migration number and generate the views migration | ||
| generateViewsMigration(Date.now()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| module.exports = class Admin1000000000000 { | ||
| name = 'Admin1000000000000' | ||
|
|
||
| async up(db) { | ||
| // Create a new "admin" schema through which the "hidden" entities can be accessed | ||
| await db.query(`CREATE SCHEMA "admin"`) | ||
| // Create admin user with "admin" schema in default "search_path" | ||
| await db.query(`CREATE USER "${process.env.DB_ADMIN_USER}" WITH PASSWORD '${process.env.DB_ADMIN_PASS}'`) | ||
| await db.query(`GRANT pg_read_all_data TO "${process.env.DB_ADMIN_USER}"`) | ||
| await db.query(`GRANT pg_write_all_data TO "${process.env.DB_ADMIN_USER}"`) | ||
| await db.query(`ALTER USER "${process.env.DB_ADMIN_USER}" SET search_path TO admin,public`) | ||
| } | ||
|
|
||
| async down(db) { | ||
| await db.query(`DROP SCHEMA "admin"`) | ||
| } | ||
| } |
Oops, something went wrong.