Wrapped data generator data creation in transaction function
ref PROD-233 Errors were not handled properly because of a missing rollback and try/catch. Using a function is easier generally. Also added ignored contraint checks to increase performance a tiny bit. They ended up not mattering much, so we can consider to remove them again.
This commit is contained in:
Simon Backx
Simon Backx
parent
eebf0e2eaf
commit
52566dc1e0
@ -149,101 +149,107 @@ class DataGenerator {
|
||||
}
|
||||
|
||||
async importData() {
|
||||
const transaction = await this.knex.transaction();
|
||||
await this.knex.transaction(async (transaction) => {
|
||||
// Performance improvements
|
||||
await this.knex.raw('SET FOREIGN_KEY_CHECKS=0;').transacting(transaction);
|
||||
await this.knex.raw('SET unique_checks=0;').transacting(transaction);
|
||||
|
||||
// Add default tables if none are specified
|
||||
if (this.tableList.length === 0) {
|
||||
this.tableList = Object.keys(importers).map(name => ({name}));
|
||||
} else if (this.withDefault) {
|
||||
// Add default tables to the end of the list
|
||||
const defaultTables = Object.keys(importers).map(name => ({name}));
|
||||
for (const table of defaultTables) {
|
||||
if (!this.tableList.find(t => t.name === table.name)) {
|
||||
this.tableList.push(table);
|
||||
// Add default tables if none are specified
|
||||
if (this.tableList.length === 0) {
|
||||
this.tableList = Object.keys(importers).map(name => ({name}));
|
||||
} else if (this.withDefault) {
|
||||
// Add default tables to the end of the list
|
||||
const defaultTables = Object.keys(importers).map(name => ({name}));
|
||||
for (const table of defaultTables) {
|
||||
if (!this.tableList.find(t => t.name === table.name)) {
|
||||
this.tableList.push(table);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Error if we have an unknown table
|
||||
for (const table of this.tableList) {
|
||||
if (importers[table.name] === undefined) {
|
||||
// eslint-disable-next-line
|
||||
throw new Error(`Unknown table: ${table.name}`);
|
||||
}
|
||||
}
|
||||
|
||||
this.sortTableList();
|
||||
|
||||
if (this.printDependencies) {
|
||||
this.logger.info('Table dependencies:');
|
||||
// Error if we have an unknown table
|
||||
for (const table of this.tableList) {
|
||||
this.logger.info(`\t${table.name}: ${table.dependencies.join(', ')}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (this.willClearData) {
|
||||
await this.clearData(transaction);
|
||||
}
|
||||
|
||||
if (this.useBaseDataPack) {
|
||||
await this.importBasePack(transaction);
|
||||
}
|
||||
|
||||
// Set quantities for tables
|
||||
for (const table of this.tableList) {
|
||||
if (this.quantities[table.name] !== undefined) {
|
||||
table.quantity = this.quantities[table.name];
|
||||
}
|
||||
}
|
||||
|
||||
const cryptoRandomBytes = crypto.randomBytes;
|
||||
|
||||
if (this.seed) {
|
||||
// The probality distributions library uses crypto.randomBytes, which we can't seed, so we need to override it
|
||||
crypto.randomBytes = (size) => {
|
||||
const buffer = Buffer.alloc(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
buffer[i] = Math.floor(faker.datatype.number({min: 0, max: 255}));
|
||||
if (importers[table.name] === undefined) {
|
||||
// eslint-disable-next-line
|
||||
throw new Error(`Unknown table: ${table.name}`);
|
||||
}
|
||||
return buffer;
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
this.sortTableList();
|
||||
|
||||
if (this.printDependencies) {
|
||||
this.logger.info('Table dependencies:');
|
||||
for (const table of this.tableList) {
|
||||
this.logger.info(`\t${table.name}: ${table.dependencies.join(', ')}`);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (this.willClearData) {
|
||||
await this.clearData(transaction);
|
||||
}
|
||||
|
||||
if (this.useBaseDataPack) {
|
||||
await this.importBasePack(transaction);
|
||||
}
|
||||
|
||||
// Set quantities for tables
|
||||
for (const table of this.tableList) {
|
||||
if (this.quantities[table.name] !== undefined) {
|
||||
table.quantity = this.quantities[table.name];
|
||||
}
|
||||
}
|
||||
|
||||
const cryptoRandomBytes = crypto.randomBytes;
|
||||
|
||||
if (this.seed) {
|
||||
// The probality distributions library uses crypto.randomBytes, which we can't seed, so we need to override it
|
||||
crypto.randomBytes = (size) => {
|
||||
const buffer = Buffer.alloc(size);
|
||||
for (let i = 0; i < size; i++) {
|
||||
buffer[i] = Math.floor(faker.datatype.number({min: 0, max: 255}));
|
||||
}
|
||||
return buffer;
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
for (const table of this.tableList) {
|
||||
if (this.seed) {
|
||||
// We reset the seed for every table, so the chosen tables don't affect the data and changes in one importer don't affect the others
|
||||
faker.seed(this.seed);
|
||||
americanFaker.seed(this.seed);
|
||||
}
|
||||
|
||||
// Add all common options to every importer, whether they use them or not
|
||||
const tableImporter = new table.importer(this.knex, transaction, {
|
||||
baseUrl: this.baseUrl
|
||||
});
|
||||
|
||||
const amount = table.quantity ?? tableImporter.defaultQuantity;
|
||||
this.logger.info('Importing content for table', table.name, amount ? `(${amount} records)` : '');
|
||||
|
||||
await tableImporter.import(table.quantity ?? undefined);
|
||||
}
|
||||
} finally {
|
||||
if (this.seed) {
|
||||
// We reset the seed for every table, so the chosen tables don't affect the data and changes in one importer don't affect the others
|
||||
faker.seed(this.seed);
|
||||
americanFaker.seed(this.seed);
|
||||
// Revert crypto.randomBytes to the original function
|
||||
crypto.randomBytes = cryptoRandomBytes;
|
||||
}
|
||||
}
|
||||
|
||||
// Add all common options to every importer, whether they use them or not
|
||||
// Finalise all tables - uses new table importer objects to avoid keeping all data in memory
|
||||
for (const table of this.tableList) {
|
||||
const tableImporter = new table.importer(this.knex, transaction, {
|
||||
baseUrl: this.baseUrl
|
||||
});
|
||||
|
||||
const amount = table.quantity ?? tableImporter.defaultQuantity;
|
||||
this.logger.info('Importing content for table', table.name, amount ? `(${amount} records)` : '');
|
||||
|
||||
await tableImporter.import(table.quantity ?? undefined);
|
||||
await tableImporter.finalise();
|
||||
}
|
||||
} finally {
|
||||
if (this.seed) {
|
||||
// Revert crypto.randomBytes to the original function
|
||||
crypto.randomBytes = cryptoRandomBytes;
|
||||
}
|
||||
}
|
||||
|
||||
// Finalise all tables - uses new table importer objects to avoid keeping all data in memory
|
||||
for (const table of this.tableList) {
|
||||
const tableImporter = new table.importer(this.knex, transaction, {
|
||||
baseUrl: this.baseUrl
|
||||
});
|
||||
await tableImporter.finalise();
|
||||
}
|
||||
|
||||
await transaction.commit();
|
||||
// Performance improvements
|
||||
await this.knex.raw('SET FOREIGN_KEY_CHECKS=1;').transacting(transaction);
|
||||
await this.knex.raw('SET unique_checks=1;').transacting(transaction);
|
||||
}, {isolationLevel: 'read committed'});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user