From 52566dc1e05766a699633c865fef920216833ac1 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Tue, 12 Dec 2023 17:36:00 +0100 Subject: [PATCH] Wrapped data generator data creation in transaction function ref PROD-233 Errors were not handled properly because of a missing rollback and try/catch. Using a function is easier generally. Also added ignored contraint checks to increase performance a tiny bit. They ended up not mattering much, so we can consider to remove them again. --- ghost/data-generator/lib/DataGenerator.js | 164 +++++++++++----------- 1 file changed, 85 insertions(+), 79 deletions(-) diff --git a/ghost/data-generator/lib/DataGenerator.js b/ghost/data-generator/lib/DataGenerator.js index 066adc8cc2..a8fdba86b8 100644 --- a/ghost/data-generator/lib/DataGenerator.js +++ b/ghost/data-generator/lib/DataGenerator.js @@ -149,101 +149,107 @@ class DataGenerator { } async importData() { - const transaction = await this.knex.transaction(); + await this.knex.transaction(async (transaction) => { + // Performance improvements + await this.knex.raw('SET FOREIGN_KEY_CHECKS=0;').transacting(transaction); + await this.knex.raw('SET unique_checks=0;').transacting(transaction); - // Add default tables if none are specified - if (this.tableList.length === 0) { - this.tableList = Object.keys(importers).map(name => ({name})); - } else if (this.withDefault) { - // Add default tables to the end of the list - const defaultTables = Object.keys(importers).map(name => ({name})); - for (const table of defaultTables) { - if (!this.tableList.find(t => t.name === table.name)) { - this.tableList.push(table); + // Add default tables if none are specified + if (this.tableList.length === 0) { + this.tableList = Object.keys(importers).map(name => ({name})); + } else if (this.withDefault) { + // Add default tables to the end of the list + const defaultTables = Object.keys(importers).map(name => ({name})); + for (const table of defaultTables) { + if (!this.tableList.find(t => t.name === table.name)) { + this.tableList.push(table); + } } } - } - // Error if we have an unknown table - for (const table of this.tableList) { - if (importers[table.name] === undefined) { - // eslint-disable-next-line - throw new Error(`Unknown table: ${table.name}`); - } - } - - this.sortTableList(); - - if (this.printDependencies) { - this.logger.info('Table dependencies:'); + // Error if we have an unknown table for (const table of this.tableList) { - this.logger.info(`\t${table.name}: ${table.dependencies.join(', ')}`); - } - process.exit(0); - } - - if (this.willClearData) { - await this.clearData(transaction); - } - - if (this.useBaseDataPack) { - await this.importBasePack(transaction); - } - - // Set quantities for tables - for (const table of this.tableList) { - if (this.quantities[table.name] !== undefined) { - table.quantity = this.quantities[table.name]; - } - } - - const cryptoRandomBytes = crypto.randomBytes; - - if (this.seed) { - // The probality distributions library uses crypto.randomBytes, which we can't seed, so we need to override it - crypto.randomBytes = (size) => { - const buffer = Buffer.alloc(size); - for (let i = 0; i < size; i++) { - buffer[i] = Math.floor(faker.datatype.number({min: 0, max: 255})); + if (importers[table.name] === undefined) { + // eslint-disable-next-line + throw new Error(`Unknown table: ${table.name}`); } - return buffer; - }; - } + } - try { + this.sortTableList(); + + if (this.printDependencies) { + this.logger.info('Table dependencies:'); + for (const table of this.tableList) { + this.logger.info(`\t${table.name}: ${table.dependencies.join(', ')}`); + } + process.exit(0); + } + + if (this.willClearData) { + await this.clearData(transaction); + } + + if (this.useBaseDataPack) { + await this.importBasePack(transaction); + } + + // Set quantities for tables for (const table of this.tableList) { + if (this.quantities[table.name] !== undefined) { + table.quantity = this.quantities[table.name]; + } + } + + const cryptoRandomBytes = crypto.randomBytes; + + if (this.seed) { + // The probality distributions library uses crypto.randomBytes, which we can't seed, so we need to override it + crypto.randomBytes = (size) => { + const buffer = Buffer.alloc(size); + for (let i = 0; i < size; i++) { + buffer[i] = Math.floor(faker.datatype.number({min: 0, max: 255})); + } + return buffer; + }; + } + + try { + for (const table of this.tableList) { + if (this.seed) { + // We reset the seed for every table, so the chosen tables don't affect the data and changes in one importer don't affect the others + faker.seed(this.seed); + americanFaker.seed(this.seed); + } + + // Add all common options to every importer, whether they use them or not + const tableImporter = new table.importer(this.knex, transaction, { + baseUrl: this.baseUrl + }); + + const amount = table.quantity ?? tableImporter.defaultQuantity; + this.logger.info('Importing content for table', table.name, amount ? `(${amount} records)` : ''); + + await tableImporter.import(table.quantity ?? undefined); + } + } finally { if (this.seed) { - // We reset the seed for every table, so the chosen tables don't affect the data and changes in one importer don't affect the others - faker.seed(this.seed); - americanFaker.seed(this.seed); + // Revert crypto.randomBytes to the original function + crypto.randomBytes = cryptoRandomBytes; } + } - // Add all common options to every importer, whether they use them or not + // Finalise all tables - uses new table importer objects to avoid keeping all data in memory + for (const table of this.tableList) { const tableImporter = new table.importer(this.knex, transaction, { baseUrl: this.baseUrl }); - - const amount = table.quantity ?? tableImporter.defaultQuantity; - this.logger.info('Importing content for table', table.name, amount ? `(${amount} records)` : ''); - - await tableImporter.import(table.quantity ?? undefined); + await tableImporter.finalise(); } - } finally { - if (this.seed) { - // Revert crypto.randomBytes to the original function - crypto.randomBytes = cryptoRandomBytes; - } - } - // Finalise all tables - uses new table importer objects to avoid keeping all data in memory - for (const table of this.tableList) { - const tableImporter = new table.importer(this.knex, transaction, { - baseUrl: this.baseUrl - }); - await tableImporter.finalise(); - } - - await transaction.commit(); + // Performance improvements + await this.knex.raw('SET FOREIGN_KEY_CHECKS=1;').transacting(transaction); + await this.knex.raw('SET unique_checks=1;').transacting(transaction); + }, {isolationLevel: 'read committed'}); } }