OpenSourceArk/Ghost
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added option to disable fast import for data generator

Browse Source 
Data generator uses CSV imports for a massive speed increase, but
can't be used in some environments where SQL admin isn't
available. This allows us to set a flag to use the original
insert-based importer.
This commit is contained in:
Sam Lord 2024-05-09 15:22:15 +01:00
parent 56d984f05f
commit 8c3e5ece01
3 changed files with 15 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
ghost/data-generator/lib/DataGenerator.js
View File

@ -204,10 +204,15 @@ class DataGenerator {
async #run(transaction) {
if (!DatabaseInfo.isSQLite(this.knex)) {
await transaction.raw('ALTER INSTANCE DISABLE INNODB REDO_LOG;');
await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
await transaction.raw('SET unique_checks=0;');
await transaction.raw('SET GLOBAL local_infile=1;');
if (process.env.DISABLE_FAST_IMPORT) {
await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
await transaction.raw('SET unique_checks=0;');
} else {
await transaction.raw('ALTER INSTANCE DISABLE INNODB REDO_LOG;');
await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
await transaction.raw('SET unique_checks=0;');
await transaction.raw('SET GLOBAL local_infile=1;');
}
}
if (this.willClearData) {
@ -274,7 +279,7 @@ class DataGenerator {
// Re-enable the redo log because it's a persisted global
// Leaving it disabled can break the database in the event of an unexpected shutdown
// See https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-disable-redo-logging
if (!DatabaseInfo.isSQLite(this.knex)) {
if (!DatabaseInfo.isSQLite(this.knex) && !process.env.DISABLE_FAST_IMPORT) {
await transaction.raw('ALTER INSTANCE ENABLE INNODB REDO_LOG;');
}
}

2
ghost/data-generator/lib/importers/TableImporter.js
View File

@ -101,7 +101,7 @@ class TableImporter {
const filePath = path.join(rootFolder, `${this.name}.csv`);
let now = Date.now();
if (data.length > 5000) {
if (data.length > 5000 && !process.env.DISABLE_FAST_IMPORT) {
try {
await fs.promises.unlink(filePath);
} catch (e) {

5
ghost/data-generator/lib/utils/JsonImporter.js
View File

@ -1,5 +1,8 @@
class JsonImporter {
const TableImporter = require('../importers/TableImporter');
class JsonImporter extends TableImporter {
constructor(knex, transaction) {
super();
this.knex = knex;
this.transaction = transaction;
}