diff --git a/ghost/core/core/cli/generate-data.js b/ghost/core/core/cli/generate-data.js index fe01eaa756..cd0f3cebb5 100644 --- a/ghost/core/core/cli/generate-data.js +++ b/ghost/core/core/cli/generate-data.js @@ -6,10 +6,10 @@ module.exports = class DataGeneratorCommand extends Command { setup() { this.help('Generates random data to populate the database for development & testing'); this.argument('--base-data-pack', {type: 'string', defaultValue: '', desc: 'Base data pack file location, imported instead of random content'}); - this.argument('--scale', {type: 'string', defaultValue: 'small', desc: 'Scale of the data to generate. `small` for a quick run, `large` for more content'}); - this.argument('--single-table', {type: 'string', desc: 'Import a single table'}); this.argument('--quantity', {type: 'number', desc: 'When importing a single table, the quantity to import'}); this.argument('--clear-database', {type: 'boolean', defaultValue: false, desc: 'Clear all entries in the database before importing'}); + this.argument('--tables', {type: 'string', desc: 'Only import the specified list of tables, where quantities can be specified by appending a colon followed by the quantity for each table. Example: --tables=members:1000,posts,tags,members_login_events'}); + this.argument('--with-default', {type: 'boolean', defaultValue: false, desc: 'Include default tables as well as those specified (simply override quantities)'}); } initializeContext(context) { @@ -30,22 +30,15 @@ module.exports = class DataGeneratorCommand extends Command { async handle(argv = {}) { const knex = require('../server/data/db/connection'); - const {tables: schema} = require('../server/data/schema/index'); - const modelQuantities = {}; - if (argv.scale) { - if (argv.scale === 'small') { - modelQuantities.members = 200; - modelQuantities.membersLoginEvents = 1; - modelQuantities.posts = 10; - } - // Defaults in data-generator package make a large set - } + const tables = (argv.tables ? argv.tables.split(',') : []).map(table => ({ + name: table.split(':')[0], + quantity: parseInt(table.split(':')[1]) || undefined + })); const dataGenerator = new DataGenerator({ baseDataPack: argv['base-data-pack'], knex, - schema, logger: { log: this.log, ok: this.ok, @@ -55,16 +48,13 @@ module.exports = class DataGeneratorCommand extends Command { fatal: this.fatal, debug: this.debug }, - modelQuantities, baseUrl: config.getSiteUrl(), - clearDatabase: argv['clear-database'] + clearDatabase: argv['clear-database'], + tables, + withDefault: argv['with-default'] }); try { - if (argv['single-table']) { - await dataGenerator.importSingleTable(argv['single-table'], argv.quantity ?? undefined); - } else { - await dataGenerator.importData(); - } + await dataGenerator.importData(); } catch (error) { this.fatal('Failed while generating data: ', error); } diff --git a/ghost/data-generator/index.js b/ghost/data-generator/index.js index c26abb561d..191d02c189 100644 --- a/ghost/data-generator/index.js +++ b/ghost/data-generator/index.js @@ -1 +1 @@ -module.exports = require('./lib/data-generator'); +module.exports = require('./lib/DataGenerator'); diff --git a/ghost/data-generator/lib/DataGenerator.js b/ghost/data-generator/lib/DataGenerator.js new file mode 100644 index 0000000000..62884ec9e0 --- /dev/null +++ b/ghost/data-generator/lib/DataGenerator.js @@ -0,0 +1,190 @@ +const path = require('path'); +const fs = require('fs/promises'); +const JsonImporter = require('./utils/JsonImporter'); +const {getProcessRoot} = require('@tryghost/root-utils'); +const topologicalSort = require('./utils/topological-sort'); + +const importers = require('./importers').reduce((acc, val) => { + acc[val.table] = val; + return acc; +}, {}); +const schema = require('../../core/core/server/data/schema').tables; + +class DataGenerator { + constructor({ + knex, + tables, + clearDatabase = false, + baseDataPack = '', + baseUrl, + logger, + withDefault + }) { + this.knex = knex; + this.tableList = tables || []; + this.willClearData = clearDatabase; + this.useBaseDataPack = baseDataPack !== ''; + this.baseDataPack = baseDataPack; + this.baseUrl = baseUrl; + this.logger = logger; + this.withDefault = withDefault; + } + + sortTableList() { + // Add missing dependencies + for (const table of this.tableList) { + table.importer = importers[table.name]; + // eslint-disable-next-line no-unused-vars + table.dependencies = Object.entries(schema[table.name]).reduce((acc, [_col, data]) => { + if (data.references) { + const referencedTable = data.references.split('.')[0]; + if (!acc.includes(referencedTable)) { + acc.push(referencedTable); + } + } + return acc; + }, table.importer.dependencies); + + for (const dependency of table.dependencies) { + if (!this.tableList.find(t => t.name === dependency)) { + this.tableList.push({ + name: dependency, + importer: importers[dependency] + }); + } + } + } + + // Order to ensure dependencies are created before dependants + this.tableList = topologicalSort(this.tableList); + } + + /** + * TODO: This needs to reverse through all dependency chains to clear data from all tables + * @param {import('knex/types').Knex.Transaction} transaction + */ + async clearData(transaction) { + const tables = this.tableList.map(t => t.name).reverse(); + + // TODO: Remove this once we import posts_meta + tables.unshift('posts_meta'); + + // Clear data from any tables that are being imported + for (const table of tables) { + this.logger.debug(`Clearing table ${table}`); + + if (table === 'roles_users') { + await transaction(table).del().whereNot('user_id', '1'); + } else if (table === 'users') { + // Avoid deleting the admin user + await transaction(table).del().whereNot('id', '1'); + } else { + await transaction(table).del(); + } + } + } + + async importBasePack(transaction) { + let baseDataPack = this.baseDataPack; + if (!path.isAbsolute(this.baseDataPack)) { + baseDataPack = path.join(getProcessRoot(), baseDataPack); + } + let baseData = {}; + try { + baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString()); + this.logger.info('Read base data pack'); + } catch (error) { + this.logger.error('Failed to read data pack: ', error); + throw error; + } + + this.logger.info('Starting base data import'); + const jsonImporter = new JsonImporter(transaction); + + // Clear settings table + await transaction('settings').del(); + + // Hard-coded for order + const tablesToImport = [ + 'newsletters', + 'posts', + 'tags', + 'products', + 'benefits', + 'products_benefits', + 'stripe_products', + 'stripe_prices', + 'settings', + 'custom_theme_settings' + ]; + for (const table of tablesToImport) { + this.logger.info(`Importing content for table ${table} from base data pack`); + await jsonImporter.import({ + name: table, + data: baseData[table] + }); + const tableIndex = this.tableList.findIndex(t => t.name === table); + if (tableIndex !== -1) { + this.tableList.splice(tableIndex, 1); + } + } + + this.logger.info('Completed base data import'); + } + + async importData() { + const transaction = await this.knex.transaction(); + + // Add default tables if none are specified + if (this.tableList.length === 0) { + this.tableList = Object.keys(importers).map(name => ({name})); + } else if (this.withDefault) { + // Add default tables to the end of the list + const defaultTables = Object.keys(importers).map(name => ({name})); + for (const table of defaultTables) { + if (!this.tableList.find(t => t.name === table.name)) { + this.tableList.push(table); + } + } + } + + // Error if we have an unknown table + for (const table of this.tableList) { + if (importers[table.name] === undefined) { + // eslint-disable-next-line + throw new Error(`Unknown table: ${table.name}`); + } + } + + this.sortTableList(); + + if (this.willClearData) { + await this.clearData(transaction); + } + + if (this.useBaseDataPack) { + await this.importBasePack(transaction); + } + + for (const table of this.tableList) { + this.logger.info('Importing content for table', table.name); + // Add all common options to every importer, whether they use them or not + const tableImporter = new table.importer(this.knex, transaction, { + baseUrl: this.baseUrl + }); + await tableImporter.import(table.quantity ?? undefined); + } + + // Finalise all tables - uses new table importer objects to avoid keeping all data in memory + for (const table of this.tableList) { + const tableImporter = new table.importer(this.knex, transaction, { + baseUrl: this.baseUrl + }); + await tableImporter.finalise(); + } + + await transaction.commit(); + } +} + +module.exports = DataGenerator; diff --git a/ghost/data-generator/lib/data-generator.js b/ghost/data-generator/lib/data-generator.js deleted file mode 100644 index 94f57fbec3..0000000000 --- a/ghost/data-generator/lib/data-generator.js +++ /dev/null @@ -1,461 +0,0 @@ -const tables = require('./tables'); -// Order here does not matter -const { - NewslettersImporter, - PostsImporter, - UsersImporter, - TagsImporter, - ProductsImporter, - MembersImporter, - BenefitsImporter, - MentionsImporter, - PostsAuthorsImporter, - PostsTagsImporter, - ProductsBenefitsImporter, - MembersProductsImporter, - PostsProductsImporter, - MembersNewslettersImporter, - StripeProductsImporter, - StripePricesImporter, - SubscriptionsImporter, - EmailsImporter, - MembersCreatedEventsImporter, - MembersLoginEventsImporter, - MembersStatusEventsImporter, - MembersSubscribeEventsImporter, - MembersStripeCustomersImporter, - MembersPaidSubscriptionEventsImporter, - EmailBatchesImporter, - EmailRecipientsImporter, - RedirectsImporter, - MembersClickEventsImporter, - OffersImporter, - MembersStripeCustomersSubscriptionsImporter, - MembersSubscriptionCreatedEventsImporter, - LabelsImporter, - MembersLabelsImporter, - RolesUsersImporter, - MembersFeedbackImporter -} = tables; -const path = require('path'); -const fs = require('fs/promises'); -const {faker} = require('@faker-js/faker'); -const JsonImporter = require('./utils/json-importer'); -const {getProcessRoot} = require('@tryghost/root-utils'); - -/** - * @typedef {Object} DataGeneratorOptions - * @property {string} baseDataPack - * @property {import('knex/types').Knex} knex - * @property {Object} schema - * @property {Object} logger - * @property {Object} modelQuantities - * @property {string} baseUrl - * @property {boolean} clearDatabase - */ - -const defaultQuantities = { - members: () => faker.datatype.number({ - min: 7000, - max: 8000 - }), - // This will generate n * events, is not worth being very high - membersLoginEvents: 5, - posts: () => faker.datatype.number({ - min: 80, - max: 120 - }) -}; - -class DataGenerator { - /** - * - * @param {DataGeneratorOptions} options - */ - constructor({ - baseDataPack = '', - knex, - schema, - logger, - modelQuantities = {}, - baseUrl, - clearDatabase - }) { - this.useBaseData = baseDataPack !== ''; - this.baseDataPack = baseDataPack; - this.knex = knex; - this.schema = schema; - this.logger = logger; - this.modelQuantities = Object.assign({}, defaultQuantities, modelQuantities); - this.baseUrl = baseUrl; - this.clearDatabase = clearDatabase; - } - - async importData() { - const transaction = await this.knex.transaction(); - - if (this.clearDatabase) { - this.logger.info('Clearing existing database'); - - // List of tables ordered to avoid dependencies when deleting - const tableNames = Object.values(tables).map(importer => importer.table).reverse(); - // We don't currently generate posts_meta, but we need to clear it to ensure posts can be removed - tableNames.unshift('posts_meta'); - for (const table of tableNames) { - this.logger.debug(`Clearing table ${table}`); - if (table === 'roles_users') { - await transaction(table).del().whereNot('user_id', '1'); - continue; - } - if (table === 'users') { - // Avoid deleting the admin user - await transaction(table).del().whereNot('id', '1'); - continue; - } - await transaction(table).del(); - } - this.logger.info('Finished clearing database'); - } - - this.logger.info('Starting import process, this has two parts: base data and member data. It can take a while...'); - - const usersImporter = new UsersImporter(transaction); - const users = await usersImporter.import({amount: 8}); - - let newsletters; - let posts; - let tags; - let products; - let stripeProducts; - let stripePrices; - let benefits; - let labels; - - // Use an existant set of data for a more realisitic looking site - if (this.useBaseData) { - let baseDataPack = this.baseDataPack; - if (!path.isAbsolute(this.baseDataPack)) { - baseDataPack = path.join(getProcessRoot(), baseDataPack); - } - let baseData = {}; - try { - baseData = JSON.parse(await (await fs.readFile(baseDataPack)).toString()); - this.logger.info('Read base data pack'); - } catch (error) { - this.logger.error('Failed to read data pack: ', error); - throw error; - } - - this.logger.info('Starting base data import'); - const jsonImporter = new JsonImporter(transaction); - - // Must have at least 2 in base data set - newsletters = await jsonImporter.import({ - name: 'newsletters', - data: baseData.newsletters, - rows: ['sort_order'] - }); - newsletters.sort((a, b) => a.sort_order - b.sort_order); - - const postsImporter = new PostsImporter(transaction, { - newsletters - }); - posts = await jsonImporter.import({ - name: 'posts', - data: baseData.posts - }); - await postsImporter.addNewsletters({posts}); - posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title').from('posts'); - - tags = await jsonImporter.import({ - name: 'tags', - data: baseData.tags - }); - - products = await jsonImporter.import({ - name: 'products', - data: baseData.products, - rows: ['name', 'monthly_price', 'yearly_price'] - }); - - benefits = await jsonImporter.import({ - name: 'benefits', - data: baseData.benefits - }); - await jsonImporter.import({ - name: 'products_benefits', - data: baseData.products_benefits - }); - - stripeProducts = await jsonImporter.import({ - name: 'stripe_products', - data: baseData.stripe_products, - rows: ['product_id', 'stripe_product_id'] - }); - stripePrices = await jsonImporter.import({ - name: 'stripe_prices', - data: baseData.stripe_prices, - rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] - }); - - labels = await jsonImporter.import({ - name: 'labels', - data: baseData.labels - }); - - // Import settings - await transaction('settings').del(); - await jsonImporter.import({ - name: 'settings', - data: baseData.settings - }); - await jsonImporter.import({ - name: 'custom_theme_settings', - data: baseData.custom_theme_settings - }); - - this.logger.info('Completed base data import'); - } else { - this.logger.info('No base data pack specified, starting random base data generation'); - const newslettersImporter = new NewslettersImporter(transaction); - // First newsletter is paid, second is free - newsletters = await newslettersImporter.import({amount: 2, rows: ['sort_order']}); - newsletters.sort((a, b) => a.sort_order - b.sort_order); - - const postsImporter = new PostsImporter(transaction, { - newsletters - }); - posts = await postsImporter.import({ - amount: this.modelQuantities.posts, - rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type'] - }); - posts.push(...await postsImporter.import({ - amount: 3, - type: 'page', - rows: ['newsletter_id', 'published_at', 'slug', 'status', 'visibility', 'title', 'type'] - })); - - const tagsImporter = new TagsImporter(transaction, { - users - }); - tags = await tagsImporter.import({amount: faker.datatype.number({ - min: 16, - max: 24 - })}); - - const productsImporter = new ProductsImporter(transaction); - products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); - - const stripeProductsImporter = new StripeProductsImporter(transaction); - stripeProducts = await stripeProductsImporter.importForEach(products.filter(product => product.name !== 'Free'), { - amount: 1, - rows: ['product_id', 'stripe_product_id'] - }); - - const stripePricesImporter = new StripePricesImporter(transaction, {products}); - stripePrices = await stripePricesImporter.importForEach(stripeProducts, { - amount: 2, - rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] - }); - - await productsImporter.addStripePrices({ - products, - stripeProducts, - stripePrices - }); - - const benefitsImporter = new BenefitsImporter(transaction); - benefits = await benefitsImporter.import({amount: 5}); - - const productsBenefitsImporter = new ProductsBenefitsImporter(transaction, {benefits}); - // Up to 5 benefits for each product - await productsBenefitsImporter.importForEach(products, {amount: 5}); - - const labelsImporter = new LabelsImporter(transaction); - labels = await labelsImporter.import({amount: 10}); - - this.logger.info('Completed random base data generation'); - } - - this.logger.info('Started member data generation'); - - const postsTagsImporter = new PostsTagsImporter(transaction, { - tags - }); - await postsTagsImporter.importForEach(posts, { - amount: () => faker.datatype.number({ - min: 0, - max: 3 - }) - }); - - const membersImporter = new MembersImporter(transaction); - const members = await membersImporter.import({amount: this.modelQuantities.members, rows: ['status', 'created_at', 'name', 'email', 'uuid']}); - - const postsAuthorsImporter = new PostsAuthorsImporter(transaction, { - users - }); - await postsAuthorsImporter.importForEach(posts, {amount: 1}); - - // TODO: Use subscriptions to generate members_products table? - const membersProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name !== 'Free')}); - const membersProducts = await membersProductsImporter.importForEach(members.filter(member => member.status !== 'free'), { - amount: 1, - rows: ['product_id', 'member_id'] - }); - const membersFreeProductsImporter = new MembersProductsImporter(transaction, {products: products.filter(product => product.name === 'Free')}); - await membersFreeProductsImporter.importForEach(members.filter(member => member.status === 'free'), { - amount: 1, - rows: ['product_id', 'member_id'] - }); - - const postsProductsImporter = new PostsProductsImporter(transaction, {products: products.slice(1)}); - // Paid newsletters - await postsProductsImporter.importForEach(posts.filter(post => newsletters.findIndex(newsletter => newsletter.id === post.newsletter_id) === 0), { - // Each post is available on all 3 premium products - amount: 3 - }); - - const membersCreatedEventsImporter = new MembersCreatedEventsImporter(transaction, {posts}); - await membersCreatedEventsImporter.importForEach(members, {amount: 1}); - - const membersLoginEventsImporter = new MembersLoginEventsImporter(transaction); - // Will create roughly 1 login event for every 3 days, up to a maximum of 100. - await membersLoginEventsImporter.importForEach(members, {amount: this.modelQuantities.membersLoginEvents}); - - const membersStatusEventsImporter = new MembersStatusEventsImporter(transaction); - // Up to 2 events per member - 1 from null -> free, 1 from free -> {paid, comped} - await membersStatusEventsImporter.importForEach(members, {amount: 2}); - - const subscriptionsImporter = new SubscriptionsImporter(transaction, {members, stripeProducts, stripePrices}); - const subscriptions = await subscriptionsImporter.importForEach(membersProducts, { - amount: 1, - rows: ['cadence', 'tier_id', 'expires_at', 'created_at', 'member_id', 'currency'] - }); - - const membersStripeCustomersImporter = new MembersStripeCustomersImporter(transaction); - const membersStripeCustomers = await membersStripeCustomersImporter.importForEach(members, { - amount: 1, - rows: ['customer_id', 'member_id'] - }); - - const membersStripeCustomersSubscriptionsImporter = new MembersStripeCustomersSubscriptionsImporter(transaction, { - membersStripeCustomers, - products, - stripeProducts, - stripePrices - }); - const membersStripeCustomersSubscriptions = await membersStripeCustomersSubscriptionsImporter.importForEach(subscriptions, { - amount: 1, - rows: ['mrr', 'plan_id', 'ghost_subscription_id'] - }); - - const membersSubscribeEventsImporter = new MembersSubscribeEventsImporter(transaction, {newsletters, subscriptions}); - const membersSubscribeEvents = await membersSubscribeEventsImporter.importForEach(members, { - amount: 2, - rows: ['member_id', 'newsletter_id', 'created_at'] - }); - - const membersNewslettersImporter = new MembersNewslettersImporter(transaction); - await membersNewslettersImporter.importForEach(membersSubscribeEvents, {amount: 1}); - - const membersPaidSubscriptionEventsImporter = new MembersPaidSubscriptionEventsImporter(transaction, { - membersStripeCustomersSubscriptions - }); - await membersPaidSubscriptionEventsImporter.importForEach(subscriptions, {amount: 1}); - - const membersSubscriptionCreatedEventsImporter = new MembersSubscriptionCreatedEventsImporter(transaction, {subscriptions, posts}); - await membersSubscriptionCreatedEventsImporter.importForEach(membersStripeCustomersSubscriptions, {amount: 1}); - - const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl}); - // Generate up to 4 webmentions per post - await mentionsImporter.importForEach(posts, {amount: 4}); - - const emailsImporter = new EmailsImporter(transaction, {newsletters, members, membersSubscribeEvents}); - const emails = await emailsImporter.importForEach(posts.filter(post => post.newsletter_id), { - amount: 1, - rows: ['created_at', 'email_count', 'delivered_count', 'opened_count', 'failed_count', 'newsletter_id', 'post_id'] - }); - - const emailBatchesImporter = new EmailBatchesImporter(transaction); - const emailBatches = await emailBatchesImporter.importForEach(emails, { - amount: 1, - rows: ['email_id', 'updated_at'] - }); - - const emailRecipientsImporter = new EmailRecipientsImporter(transaction, {emailBatches, members, membersSubscribeEvents}); - const emailRecipients = await emailRecipientsImporter.importForEach(emails, { - amount: this.modelQuantities.members, - rows: ['opened_at', 'email_id', 'member_id'] - }); - - await membersImporter.addOpenRate({emailRecipients}); - - const redirectsImporter = new RedirectsImporter(transaction); - const redirects = await redirectsImporter.importForEach(posts.filter(post => post.newsletter_id), { - amount: 10, - rows: ['post_id'] - }); - - const membersClickEventsImporter = new MembersClickEventsImporter(transaction, {redirects, emails}); - await membersClickEventsImporter.importForEach(emailRecipients, {amount: 2}); - - const offersImporter = new OffersImporter(transaction, {products: products.filter(product => product.name !== 'Free')}); - await offersImporter.import({amount: 2}); - - const membersLabelsImporter = new MembersLabelsImporter(transaction, {labels}); - await membersLabelsImporter.importForEach(members, { - amount: 1 - }); - - const roles = await transaction.select('id', 'name').from('roles'); - const rolesUsersImporter = new RolesUsersImporter(transaction, {roles}); - await rolesUsersImporter.importForEach(users, {amount: 1}); - - const membersFeedbackImporter = new MembersFeedbackImporter(transaction, {emails}); - await membersFeedbackImporter.importForEach(emailRecipients, {amount: 1}); - - await transaction.commit(); - - this.logger.info('Completed member data generation'); - this.logger.ok('Completed import process.'); - } - - async importSingleTable(tableName, quantity) { - this.logger.info('Importing a single table'); - const transaction = await this.knex.transaction(); - - const importMembers = async () => { - this.logger.info(`Importing ${quantity ?? this.modelQuantities.members} members`); - const membersImporter = new MembersImporter(transaction); - await membersImporter.import({amount: quantity ?? this.modelQuantities.members}); - }; - - const importMentions = async () => { - const posts = await transaction.select('id', 'newsletter_id', 'published_at', 'slug', 'status', 'visibility').from('posts'); - this.logger.info(`Importing up to ${posts.length * 4} mentions`); - - const mentionsImporter = new MentionsImporter(transaction, {baseUrl: this.baseUrl}); - await mentionsImporter.importForEach(posts, {amount: 4}); - }; - - switch (tableName) { - case 'members': - await importMembers(); - break; - case 'mentions': - await importMentions(); - break; - default: - this.logger.warn(`Cannot import single table '${tableName}'`); - await transaction.commit(); // no-op, just close the transaction - return; - } - - this.logger.ok('Completed import process.'); - await transaction.commit(); - } -} - -module.exports = DataGenerator; -module.exports.tables = tables; diff --git a/ghost/data-generator/lib/tables/benefits.js b/ghost/data-generator/lib/importers/BenefitsImporter.js similarity index 77% rename from ghost/data-generator/lib/tables/benefits.js rename to ghost/data-generator/lib/importers/BenefitsImporter.js index ce9bc0f25e..c0dd3f2a68 100644 --- a/ghost/data-generator/lib/tables/benefits.js +++ b/ghost/data-generator/lib/importers/BenefitsImporter.js @@ -1,13 +1,15 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const {blogStartDate} = require('../utils/blog-info'); class BenefitsImporter extends TableImporter { static table = 'benefits'; + static dependencies = []; + defaultQuantity = 5; - constructor(knex) { - super(BenefitsImporter.table, knex); + constructor(knex, transaction) { + super(BenefitsImporter.table, knex, transaction); } generate() { diff --git a/ghost/data-generator/lib/tables/email-batches.js b/ghost/data-generator/lib/importers/EmailBatchesImporter.js similarity index 69% rename from ghost/data-generator/lib/tables/email-batches.js rename to ghost/data-generator/lib/importers/EmailBatchesImporter.js index c069f8a071..b09da9a6f8 100644 --- a/ghost/data-generator/lib/tables/email-batches.js +++ b/ghost/data-generator/lib/importers/EmailBatchesImporter.js @@ -1,16 +1,20 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const dateToDatabaseString = require('../utils/database-date'); class EmailBatchesImporter extends TableImporter { static table = 'email_batches'; + static dependencies = ['emails']; - constructor(knex) { - super(EmailBatchesImporter.table, knex); + constructor(knex, transaction) { + super(EmailBatchesImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const emails = await this.transaction.select('id', 'created_at').from('emails'); + + // TODO: Generate >1 batch per email + await this.importForEach(emails, quantity ?? emails.length); } generate() { diff --git a/ghost/data-generator/lib/tables/email-recipients.js b/ghost/data-generator/lib/importers/EmailRecipientsImporter.js similarity index 77% rename from ghost/data-generator/lib/tables/email-recipients.js rename to ghost/data-generator/lib/importers/EmailRecipientsImporter.js index 4b19960523..b51383aa1a 100644 --- a/ghost/data-generator/lib/tables/email-recipients.js +++ b/ghost/data-generator/lib/importers/EmailRecipientsImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const generateEvents = require('../utils/event-generator'); const dateToDatabaseString = require('../utils/database-date'); @@ -12,15 +12,30 @@ const emailStatus = { class EmailRecipientsImporter extends TableImporter { static table = 'email_recipients'; + static dependencies = ['emails', 'email_batches', 'members', 'members_subscribe_events']; - constructor(knex, {emailBatches, members, membersSubscribeEvents}) { - super(EmailRecipientsImporter.table, knex); - this.emailBatches = emailBatches; - this.members = members; - this.membersSubscribeEvents = membersSubscribeEvents; + constructor(knex, transaction) { + super(EmailRecipientsImporter.table, knex, transaction); } - setImportOptions({model}) { + async import(quantity) { + const emails = await this.transaction + .select( + 'id', + 'newsletter_id', + 'email_count', + 'delivered_count', + 'opened_count', + 'failed_count') + .from('emails'); + this.emailBatches = await this.transaction.select('id', 'email_id', 'updated_at').from('email_batches'); + this.members = await this.transaction.select('id', 'uuid', 'email', 'name').from('members'); + this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at', 'member_id').from('members_subscribe_events'); + + await this.importForEach(emails, quantity ? quantity / emails.length : this.members.length); + } + + setReferencedModel(model) { this.model = model; this.batch = this.emailBatches.find(batch => batch.email_id === model.id); // Shallow clone members list so we can shuffle and modify it diff --git a/ghost/data-generator/lib/tables/emails.js b/ghost/data-generator/lib/importers/EmailsImporter.js similarity index 80% rename from ghost/data-generator/lib/tables/emails.js rename to ghost/data-generator/lib/importers/EmailsImporter.js index 3e3a892381..eb867ed331 100644 --- a/ghost/data-generator/lib/tables/emails.js +++ b/ghost/data-generator/lib/importers/EmailsImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const generateEvents = require('../utils/event-generator'); const {luck} = require('../utils/random'); @@ -6,16 +6,18 @@ const dateToDatabaseString = require('../utils/database-date'); class EmailsImporter extends TableImporter { static table = 'emails'; + static dependencies = ['posts', 'newsletters', 'members_subscribe_events']; - constructor(knex, {newsletters, members, membersSubscribeEvents}) { - super(EmailsImporter.table, knex); - this.newsletters = newsletters; - this.members = members; - this.membersSubscribeEvents = membersSubscribeEvents; + constructor(knex, transaction) { + super(EmailsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const posts = await this.transaction.select('id', 'title', 'published_at').from('posts').where('type', 'post'); + this.newsletters = await this.transaction.select('id').from('newsletters'); + this.membersSubscribeEvents = await this.transaction.select('id', 'newsletter_id', 'created_at').from('members_subscribe_events'); + + await this.importForEach(posts, quantity ? quantity / posts.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/labels.js b/ghost/data-generator/lib/importers/LabelsImporter.js similarity index 84% rename from ghost/data-generator/lib/tables/labels.js rename to ghost/data-generator/lib/importers/LabelsImporter.js index 70ca860cb8..d4db241a88 100644 --- a/ghost/data-generator/lib/tables/labels.js +++ b/ghost/data-generator/lib/importers/LabelsImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const {blogStartDate} = require('../utils/blog-info'); @@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date'); class LabelsImporter extends TableImporter { static table = 'labels'; + static dependencies = []; + defaultQuantity = 10; - constructor(knex) { - super(LabelsImporter.table, knex); + constructor(knex, transaction) { + super(LabelsImporter.table, knex, transaction); this.generatedNames = new Set(); } diff --git a/ghost/data-generator/lib/tables/members-click-events.js b/ghost/data-generator/lib/importers/MembersClickEventsImporter.js similarity index 61% rename from ghost/data-generator/lib/tables/members-click-events.js rename to ghost/data-generator/lib/importers/MembersClickEventsImporter.js index 5d64318116..da65f5593e 100644 --- a/ghost/data-generator/lib/tables/members-click-events.js +++ b/ghost/data-generator/lib/importers/MembersClickEventsImporter.js @@ -1,30 +1,37 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); const dateToDatabaseString = require('../utils/database-date'); class MembersClickEventsImporter extends TableImporter { static table = 'members_click_events'; + static dependencies = ['email_recipients', 'redirects', 'emails']; - constructor(knex, {redirects, emails}) { - super(MembersClickEventsImporter.table, knex); - - this.redirects = redirects; - this.emails = emails; + constructor(knex, transaction) { + super(MembersClickEventsImporter.table, knex, transaction); } - setImportOptions({model, amount}) { + async import(quantity) { + const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients'); + this.redirects = await this.transaction.select('id', 'post_id').from('redirects'); + this.emails = await this.transaction.select('id', 'post_id').from('emails'); + this.quantity = quantity ? quantity / emailRecipients.length : 2; + + await this.importForEach(emailRecipients, this.quantity); + } + + setReferencedModel(model) { this.model = model; this.amount = model.opened_at === null ? 0 : luck(40) ? faker.datatype.number({ min: 0, - max: amount + max: this.quantity }) : 0; const email = this.emails.find(e => e.id === this.model.email_id); this.redirectList = this.redirects.filter(redirect => redirect.post_id === email.post_id); } generate() { - if (this.amount <= 0) { + if (this.amount <= 0 || this.redirectList.length === 0) { return; } this.amount -= 1; diff --git a/ghost/data-generator/lib/tables/members-created-events.js b/ghost/data-generator/lib/importers/MembersCreatedEventsImporter.js similarity index 69% rename from ghost/data-generator/lib/tables/members-created-events.js rename to ghost/data-generator/lib/importers/MembersCreatedEventsImporter.js index 12e630c6da..c5131e4070 100644 --- a/ghost/data-generator/lib/tables/members-created-events.js +++ b/ghost/data-generator/lib/importers/MembersCreatedEventsImporter.js @@ -1,20 +1,20 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); class MembersCreatedEventsImporter extends TableImporter { static table = 'members_created_events'; + static dependencies = ['members', 'posts']; - constructor(knex, {posts}) { - super(MembersCreatedEventsImporter.table, knex); - - this.posts = [...posts]; - // Sort posts in reverse chronologoical order - this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf()); + constructor(knex, transaction) { + super(MembersCreatedEventsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const members = await this.transaction.select('id', 'created_at').from('members'); + this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc'); + + await this.importForEach(members, quantity ? quantity / members.length : 1); } generateSource() { diff --git a/ghost/data-generator/lib/tables/members-feedback.js b/ghost/data-generator/lib/importers/MembersFeedbackImporter.js similarity index 67% rename from ghost/data-generator/lib/tables/members-feedback.js rename to ghost/data-generator/lib/importers/MembersFeedbackImporter.js index c862159028..f8d5b5cdd1 100644 --- a/ghost/data-generator/lib/tables/members-feedback.js +++ b/ghost/data-generator/lib/importers/MembersFeedbackImporter.js @@ -1,18 +1,22 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); const dateToDatabaseString = require('../utils/database-date'); class MembersFeedbackImporter extends TableImporter { static table = 'members_feedback'; + static dependencies = ['emails', 'email_recipients']; - constructor(knex, {emails}) { - super(MembersFeedbackImporter.table, knex); + constructor(knex, transaction, {emails}) { + super(MembersFeedbackImporter.table, knex, transaction); this.emails = emails; } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const emailRecipients = await this.transaction.select('id', 'opened_at', 'email_id', 'member_id').from('email_recipients'); + this.emails = await this.transaction.select('id', 'post_id').from('emails'); + + await this.importForEach(emailRecipients, quantity ? quantity / emailRecipients.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/members.js b/ghost/data-generator/lib/importers/MembersImporter.js similarity index 85% rename from ghost/data-generator/lib/tables/members.js rename to ghost/data-generator/lib/importers/MembersImporter.js index e36f1a2026..beb1e20945 100644 --- a/ghost/data-generator/lib/tables/members.js +++ b/ghost/data-generator/lib/importers/MembersImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {faker: americanFaker} = require('@faker-js/faker/locale/en_US'); const {blogStartDate: startTime} = require('../utils/blog-info'); @@ -8,22 +8,34 @@ const dateToDatabaseString = require('../utils/database-date'); class MembersImporter extends TableImporter { static table = 'members'; + static dependencies = []; + defaultQuantity = faker.datatype.number({ + min: 7000, + max: 8000 + }); - constructor(knex) { - super(MembersImporter.table, knex); + constructor(knex, transaction) { + super(MembersImporter.table, knex, transaction); } - setImportOptions({amount}) { + async import(quantity = this.defaultQuantity) { this.timestamps = generateEvents({ shape: 'ease-in', trend: 'positive', - total: amount, + total: quantity, startTime, endTime: new Date() }).sort(); + + await super.import(quantity); } - async addOpenRate({emailRecipients}) { + /** + * Add open rate data to members table + */ + async finalise() { + const emailRecipients = await this.transaction.select('id', 'member_id', 'opened_at').from('email_recipients'); + const memberData = {}; for (const emailRecipient of emailRecipients) { if (!(emailRecipient.member_id in memberData)) { @@ -41,7 +53,7 @@ class MembersImporter extends TableImporter { for (const [memberId, emailInfo] of Object.entries(memberData)) { const openRate = Math.round(100 * (emailInfo.openedCount / emailInfo.emailCount)); - await this.knex('members').update({ + await this.transaction('members').update({ email_count: emailInfo.emailCount, email_opened_count: emailInfo.openedCount, email_open_rate: emailInfo.emailCount >= 5 ? openRate : null diff --git a/ghost/data-generator/lib/tables/members-labels.js b/ghost/data-generator/lib/importers/MembersLabelsImporter.js similarity index 55% rename from ghost/data-generator/lib/tables/members-labels.js rename to ghost/data-generator/lib/importers/MembersLabelsImporter.js index e3a3176c3a..9c45279232 100644 --- a/ghost/data-generator/lib/tables/members-labels.js +++ b/ghost/data-generator/lib/importers/MembersLabelsImporter.js @@ -1,17 +1,21 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); class MembersLabelsImporter extends TableImporter { static table = 'members_labels'; + static dependencies = ['labels', 'members']; - constructor(knex, {labels}) { - super(MembersLabelsImporter.table, knex); + constructor(knex, transaction, {labels}) { + super(MembersLabelsImporter.table, knex, transaction); this.labels = labels; } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const members = await this.transaction.select('id').from('members'); + this.labels = await this.transaction.select('id').from('labels'); + + await this.importForEach(members, quantity ? quantity / members.length : 1); } generate() { @@ -19,6 +23,7 @@ class MembersLabelsImporter extends TableImporter { // 90% of members don't have labels return; } + // TODO: Ensure we don't generate the same member label twice return { id: faker.database.mongodbObjectId(), member_id: this.model.id, diff --git a/ghost/data-generator/lib/tables/members-login-events.js b/ghost/data-generator/lib/importers/MembersLoginEventsImporter.js similarity index 77% rename from ghost/data-generator/lib/tables/members-login-events.js rename to ghost/data-generator/lib/importers/MembersLoginEventsImporter.js index 3e6c05f9d5..bfebbdd141 100644 --- a/ghost/data-generator/lib/tables/members-login-events.js +++ b/ghost/data-generator/lib/importers/MembersLoginEventsImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); const generateEvents = require('../utils/event-generator'); @@ -6,13 +6,21 @@ const dateToDatabaseString = require('../utils/database-date'); class MembersLoginEventsImporter extends TableImporter { static table = 'members_login_events'; + static dependencies = ['members']; - constructor(knex) { - super(MembersLoginEventsImporter.table, knex); + constructor(knex, transaction) { + super(MembersLoginEventsImporter.table, knex, transaction); } - setImportOptions({model}) { + async import(quantity) { + const members = await this.transaction.select('id', 'created_at').from('members'); + + await this.importForEach(members, quantity ? quantity / members.length : 5); + } + + setReferencedModel(model) { this.model = model; + const endDate = new Date(); const daysBetween = Math.ceil((endDate.valueOf() - new Date(model.created_at).valueOf()) / (1000 * 60 * 60 * 24)); diff --git a/ghost/data-generator/lib/importers/MembersNewslettersImporter.js b/ghost/data-generator/lib/importers/MembersNewslettersImporter.js new file mode 100644 index 0000000000..dc4ee0185d --- /dev/null +++ b/ghost/data-generator/lib/importers/MembersNewslettersImporter.js @@ -0,0 +1,27 @@ +const {faker} = require('@faker-js/faker'); +const TableImporter = require('./TableImporter'); + +class MembersNewslettersImporter extends TableImporter { + static table = 'members_newsletters'; + static dependencies = ['members_subscribe_events']; + + constructor(knex, transaction) { + super(MembersNewslettersImporter.table, knex, transaction); + } + + async import(quantity) { + const membersSubscribeEvents = await this.transaction.select('member_id', 'newsletter_id').from('members_subscribe_events'); + + await this.importForEach(membersSubscribeEvents, quantity ? quantity / membersSubscribeEvents.length : 1); + } + + generate() { + return { + id: faker.database.mongodbObjectId(), + member_id: this.model.member_id, + newsletter_id: this.model.newsletter_id + }; + } +} + +module.exports = MembersNewslettersImporter; diff --git a/ghost/data-generator/lib/tables/members-paid-subscription-events.js b/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js similarity index 62% rename from ghost/data-generator/lib/tables/members-paid-subscription-events.js rename to ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js index 2c035cd678..afc212f2e5 100644 --- a/ghost/data-generator/lib/tables/members-paid-subscription-events.js +++ b/ghost/data-generator/lib/importers/MembersPaidSubscriptionEventsImporter.js @@ -1,16 +1,19 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); class MembersPaidSubscriptionEventsImporter extends TableImporter { static table = 'members_paid_subscription_events'; + static dependencies = ['subscriptions', 'members_stripe_customers_subscriptions']; - constructor(knex, {membersStripeCustomersSubscriptions}) { - super(MembersPaidSubscriptionEventsImporter.table, knex); - this.membersStripeCustomersSubscriptions = membersStripeCustomersSubscriptions; + constructor(knex, transaction) { + super(MembersPaidSubscriptionEventsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const subscriptions = await this.transaction.select('id', 'member_id', 'currency', 'created_at').from('subscriptions'); + this.membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id', 'plan_id', 'mrr').from('members_stripe_customers_subscriptions'); + + await this.importForEach(subscriptions, quantity ? quantity / subscriptions.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/members-products.js b/ghost/data-generator/lib/importers/MembersProductsImporter.js similarity index 57% rename from ghost/data-generator/lib/tables/members-products.js rename to ghost/data-generator/lib/importers/MembersProductsImporter.js index 9c2044d14b..ff4a565225 100644 --- a/ghost/data-generator/lib/tables/members-products.js +++ b/ghost/data-generator/lib/importers/MembersProductsImporter.js @@ -1,16 +1,20 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {luck} = require('../utils/random'); class MembersProductsImporter extends TableImporter { static table = 'members_products'; - constructor(knex, {products}) { - super(MembersProductsImporter.table, knex); - this.products = products; + static dependencies = ['products', 'members']; + + constructor(knex, transaction) { + super(MembersProductsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const members = await this.transaction.select('id').from('members').whereNot('status', 'free'); + this.products = await this.transaction.select('id').from('products').whereNot('name', 'Free'); + + await this.importForEach(members, quantity ? quantity / members.length : 1); } getProduct() { diff --git a/ghost/data-generator/lib/tables/members-status-events.js b/ghost/data-generator/lib/importers/MembersStatusEventsImporter.js similarity index 69% rename from ghost/data-generator/lib/tables/members-status-events.js rename to ghost/data-generator/lib/importers/MembersStatusEventsImporter.js index 239b32239f..90b98f78dd 100644 --- a/ghost/data-generator/lib/tables/members-status-events.js +++ b/ghost/data-generator/lib/importers/MembersStatusEventsImporter.js @@ -1,15 +1,22 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const dateToDatabaseString = require('../utils/database-date'); class MembersStatusEventsImporter extends TableImporter { static table = 'members_status_events'; + static dependencies = ['members']; - constructor(knex) { - super(MembersStatusEventsImporter.table, knex); + constructor(knex, transaction) { + super(MembersStatusEventsImporter.table, knex, transaction); } - setImportOptions({model}) { + async import(quantity) { + const members = await this.transaction.select('id', 'created_at', 'status').from('members'); + + await this.importForEach(members, quantity ? quantity / members.length : 2); + } + + setReferencedModel(model) { this.events = [{ id: faker.database.mongodbObjectId(), member_id: model.id, diff --git a/ghost/data-generator/lib/tables/members-stripe-customers.js b/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js similarity index 59% rename from ghost/data-generator/lib/tables/members-stripe-customers.js rename to ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js index d89fd05585..832277f310 100644 --- a/ghost/data-generator/lib/tables/members-stripe-customers.js +++ b/ghost/data-generator/lib/importers/MembersStripeCustomersImporter.js @@ -1,15 +1,18 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class MembersStripeCustomersImporter extends TableImporter { static table = 'members_stripe_customers'; + static dependencies = ['members']; - constructor(knex) { - super(MembersStripeCustomersImporter.table, knex); + constructor(knex, transaction) { + super(MembersStripeCustomersImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const members = await this.transaction.select('id', 'name', 'email', 'created_at').from('members'); + + await this.importForEach(members, quantity ? quantity / members.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js b/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js similarity index 65% rename from ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js rename to ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js index 0ab8876ffd..4bf38f20a8 100644 --- a/ghost/data-generator/lib/tables/members-stripe-customers-subscriptions.js +++ b/ghost/data-generator/lib/importers/MembersStripeCustomersSubscriptionsImporter.js @@ -1,19 +1,22 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class MembersStripeCustomersSubscriptionsImporter extends TableImporter { static table = 'members_stripe_customers_subscriptions'; + static dependencies = ['subscriptions', 'members_stripe_customers', 'products', 'stripe_products', 'stripe_prices']; - constructor(knex, {membersStripeCustomers, products, stripeProducts, stripePrices}) { - super(MembersStripeCustomersSubscriptionsImporter.table, knex); - this.membersStripeCustomers = membersStripeCustomers; - this.products = products; - this.stripeProducts = stripeProducts; - this.stripePrices = stripePrices; + constructor(knex, transaction) { + super(MembersStripeCustomersSubscriptionsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import() { + const subscriptions = await this.transaction.select('id', 'member_id', 'tier_id', 'cadence', 'created_at', 'expires_at').from('subscriptions'); + this.membersStripeCustomers = await this.transaction.select('id', 'member_id', 'customer_id').from('members_stripe_customers'); + this.products = await this.transaction.select('id', 'name').from('products'); + this.stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products'); + this.stripePrices = await this.transaction.select('id', 'nickname', 'stripe_product_id', 'stripe_price_id', 'amount', 'interval', 'currency').from('stripe_prices'); + + await this.importForEach(subscriptions, 1); } generate() { diff --git a/ghost/data-generator/lib/tables/members-subscribe-events.js b/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js similarity index 67% rename from ghost/data-generator/lib/tables/members-subscribe-events.js rename to ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js index 8972eb066b..bcde10d0f0 100644 --- a/ghost/data-generator/lib/tables/members-subscribe-events.js +++ b/ghost/data-generator/lib/importers/MembersSubscribeEventsImporter.js @@ -1,17 +1,25 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); const dateToDatabaseString = require('../utils/database-date'); class MembersSubscribeEventsImporter extends TableImporter { static table = 'members_subscribe_events'; - constructor(knex, {newsletters, subscriptions}) { - super(MembersSubscribeEventsImporter.table, knex); - this.newsletters = newsletters; - this.subscriptions = subscriptions; + static dependencies = ['members', 'newsletters', 'subscriptions']; + + constructor(knex, transaction) { + super(MembersSubscribeEventsImporter.table, knex, transaction); } - setImportOptions({model}) { + async import(quantity) { + const members = await this.transaction.select('id', 'created_at', 'status').from('members'); + this.newsletters = await this.transaction.select('id').from('newsletters'); + this.subscriptions = await this.transaction.select('member_id', 'created_at').from('subscriptions'); + + await this.importForEach(members, quantity ? quantity / members.length : 2); + } + + setReferencedModel(model) { this.model = model; this.count = 0; } diff --git a/ghost/data-generator/lib/tables/members-subscription-created-events.js b/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js similarity index 62% rename from ghost/data-generator/lib/tables/members-subscription-created-events.js rename to ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js index 378f23ac82..081436a5cd 100644 --- a/ghost/data-generator/lib/tables/members-subscription-created-events.js +++ b/ghost/data-generator/lib/importers/MembersSubscriptionCreatedEventsImporter.js @@ -1,21 +1,21 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {luck} = require('../utils/random'); class MembersSubscriptionCreatedEventsImporter extends TableImporter { static table = 'members_subscription_created_events'; + static dependencies = ['members_stripe_customers_subscriptions', 'subscriptions', 'posts']; - constructor(knex, {subscriptions, posts}) { - super(MembersSubscriptionCreatedEventsImporter.table, knex); - this.subscriptions = subscriptions; - - this.posts = [...posts]; - // Sort posts in reverse chronologoical order - this.posts.sort((a, b) => new Date(b.published_at).valueOf() - new Date(a.published_at).valueOf()); + constructor(knex, transaction) { + super(MembersSubscriptionCreatedEventsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const membersStripeCustomersSubscriptions = await this.transaction.select('id', 'ghost_subscription_id').from('members_stripe_customers_subscriptions'); + this.subscriptions = await this.transaction.select('id', 'created_at', 'member_id').from('subscriptions'); + this.posts = await this.transaction.select('id', 'published_at', 'visibility', 'type', 'slug').from('posts').orderBy('published_at', 'desc'); + + await this.importForEach(membersStripeCustomersSubscriptions, quantity ? quantity / membersStripeCustomersSubscriptions.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/newsletters.js b/ghost/data-generator/lib/importers/NewslettersImporter.js similarity index 79% rename from ghost/data-generator/lib/tables/newsletters.js rename to ghost/data-generator/lib/importers/NewslettersImporter.js index 7d17093e66..f8b0011c53 100644 --- a/ghost/data-generator/lib/tables/newsletters.js +++ b/ghost/data-generator/lib/importers/NewslettersImporter.js @@ -1,14 +1,17 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {blogStartDate} = require('../utils/blog-info'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); class NewslettersImporter extends TableImporter { static table = 'newsletters'; + static dependencies = []; + defaultQuantity = 2; - constructor(knex) { - super(NewslettersImporter.table, knex); + constructor(knex, transaction) { + super(NewslettersImporter.table, knex, transaction); this.sortOrder = 0; + // TODO: Use random names if we ever need more than 2 newsletters this.names = ['Regular premium', 'Occasional freebie']; } diff --git a/ghost/data-generator/lib/tables/offers.js b/ghost/data-generator/lib/importers/OffersImporter.js similarity index 88% rename from ghost/data-generator/lib/tables/offers.js rename to ghost/data-generator/lib/importers/OffersImporter.js index f326bda3dc..3ab55c268d 100644 --- a/ghost/data-generator/lib/tables/offers.js +++ b/ghost/data-generator/lib/importers/OffersImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const {blogStartDate} = require('../utils/blog-info'); @@ -6,15 +6,19 @@ const dateToDatabaseString = require('../utils/database-date'); class OffersImporter extends TableImporter { static table = 'offers'; + static dependencies = ['products']; + defaultQuantity = 2; - constructor(knex, {products}) { - super(OffersImporter.table, knex); - this.products = products; + constructor(knex, transaction) { + super(OffersImporter.table, knex, transaction); } - setImportOptions() { + async import(quantity = this.defaultQuantity) { + this.products = await this.transaction.select('id', 'currency').from('products'); this.names = ['Black Friday', 'Free Trial']; this.count = 0; + + await super.import(quantity); } generate() { diff --git a/ghost/data-generator/lib/tables/posts-authors.js b/ghost/data-generator/lib/importers/PostsAuthorsImporter.js similarity index 55% rename from ghost/data-generator/lib/tables/posts-authors.js rename to ghost/data-generator/lib/importers/PostsAuthorsImporter.js index 72bdc9ff5a..4d714b240a 100644 --- a/ghost/data-generator/lib/tables/posts-authors.js +++ b/ghost/data-generator/lib/importers/PostsAuthorsImporter.js @@ -1,17 +1,20 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class PostsAuthorsImporter extends TableImporter { static table = 'posts_authors'; + static dependencies = ['posts', 'users']; - constructor(knex, {users}) { - super(PostsAuthorsImporter.table, knex); - this.users = users; + constructor(knex, transaction) { + super(PostsAuthorsImporter.table, knex, transaction); this.sortOrder = 0; } - setImportOptions({model}) { - this.model = model; + async import(quantity) { + const posts = await this.transaction.select('id').from('posts'); + this.users = await this.transaction.select('id').from('users'); + + await this.importForEach(posts, quantity ? quantity / posts.length : 1); } generate() { diff --git a/ghost/data-generator/lib/tables/posts.js b/ghost/data-generator/lib/importers/PostsImporter.js similarity index 82% rename from ghost/data-generator/lib/tables/posts.js rename to ghost/data-generator/lib/importers/PostsImporter.js index 17111d2f7f..f0844f6d42 100644 --- a/ghost/data-generator/lib/tables/posts.js +++ b/ghost/data-generator/lib/importers/PostsImporter.js @@ -1,27 +1,27 @@ const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const {luck} = require('../utils/random'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const dateToDatabaseString = require('../utils/database-date'); class PostsImporter extends TableImporter { static table = 'posts'; + static dependencies = ['newsletters']; + defaultQuantity = faker.datatype.number({ + min: 80, + max: 120 + }); - constructor(knex, {newsletters}) { - super(PostsImporter.table, knex); - this.newsletters = newsletters; + type = 'post'; + + constructor(knex, transaction) { + super(PostsImporter.table, knex, transaction); } - setImportOptions({type = 'post'}) { - this.type = type; - } + async import(quantity = this.defaultQuantity) { + this.newsletters = await this.transaction.select('id').from('newsletters'); - async addNewsletters({posts}) { - for (const {id, visibility} of posts) { - await this.knex('posts').update({ - newsletter_id: luck(90) ? (visibility === 'paid' ? this.newsletters[0].id : this.newsletters[1].id) : null - }).where({id, type: 'post', status: 'published'}); - } + await super.import(quantity); } generate() { diff --git a/ghost/data-generator/lib/tables/posts-products.js b/ghost/data-generator/lib/importers/PostsProductsImporter.js similarity index 51% rename from ghost/data-generator/lib/tables/posts-products.js rename to ghost/data-generator/lib/importers/PostsProductsImporter.js index 1d799c85cb..8a223c684c 100644 --- a/ghost/data-generator/lib/tables/posts-products.js +++ b/ghost/data-generator/lib/importers/PostsProductsImporter.js @@ -1,17 +1,24 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class PostsProductsImporter extends TableImporter { static table = 'posts_products'; + static dependencies = ['posts', 'products']; - constructor(knex, {products}) { - super(PostsProductsImporter.table, knex); - this.products = products; + constructor(knex, transaction) { + super(PostsProductsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.sortOrder = 0; + async import(quantity) { + const posts = await this.transaction.select('id').from('posts').where('type', 'post'); + this.products = await this.transaction.select('id').from('products'); + + await this.importForEach(posts, quantity ? quantity / posts.length : 1); + } + + setReferencedModel(model) { this.model = model; + this.sortOrder = 0; } generate() { diff --git a/ghost/data-generator/lib/tables/posts-tags.js b/ghost/data-generator/lib/importers/PostsTagsImporter.js similarity index 56% rename from ghost/data-generator/lib/tables/posts-tags.js rename to ghost/data-generator/lib/importers/PostsTagsImporter.js index 116b0b2f4d..b9fcbe3e7d 100644 --- a/ghost/data-generator/lib/tables/posts-tags.js +++ b/ghost/data-generator/lib/importers/PostsTagsImporter.js @@ -1,18 +1,28 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class PostsTagsImporter extends TableImporter { static table = 'posts_tags'; - constructor(knex, {tags}) { - super(PostsTagsImporter.table, knex); - this.tags = tags; + static dependencies = ['posts', 'tags']; + + constructor(knex, transaction) { + super(PostsTagsImporter.table, knex, transaction); this.sortOrder = 0; } - setImportOptions({model}) { - this.notIndex = []; - this.sortOrder = 0; + async import(quantity) { + const posts = await this.transaction.select('id').from('posts').where('type', 'post'); + this.tags = await this.transaction.select('id').from('tags'); + + await this.importForEach(posts, quantity ? quantity / posts.length : () => faker.datatype.number({ + min: 0, + max: 3 + })); + } + + setReferencedModel(model) { this.model = model; + this.notIndex = []; } generate() { diff --git a/ghost/data-generator/lib/tables/products-benefits.js b/ghost/data-generator/lib/importers/ProductsBenefitsImporter.js similarity index 66% rename from ghost/data-generator/lib/tables/products-benefits.js rename to ghost/data-generator/lib/importers/ProductsBenefitsImporter.js index b3c37e6f23..58cff71b04 100644 --- a/ghost/data-generator/lib/tables/products-benefits.js +++ b/ghost/data-generator/lib/importers/ProductsBenefitsImporter.js @@ -1,17 +1,24 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class ProductsBenefitsImporter extends TableImporter { static table = 'products_benefits'; + static dependencies = ['benefits', 'products']; - constructor(knex, {benefits}) { - super(ProductsBenefitsImporter.table, knex); - this.benefits = benefits; - this.sortOrder = 0; + constructor(knex, transaction) { + super(ProductsBenefitsImporter.table, knex, transaction); } - setImportOptions({model}) { + async import(quantity) { + const products = await this.transaction.select('id', 'name').from('products'); + this.benefits = await this.transaction.select('id').from('benefits'); + + await this.importForEach(products, quantity ? quantity / products.length : 5); + } + + setReferencedModel(model) { this.model = model; + this.sortOrder = 0; switch (this.model.name) { case 'Bronze': diff --git a/ghost/data-generator/lib/tables/products.js b/ghost/data-generator/lib/importers/ProductsImporter.js similarity index 71% rename from ghost/data-generator/lib/tables/products.js rename to ghost/data-generator/lib/importers/ProductsImporter.js index 0cfec94fa3..6ad6102873 100644 --- a/ghost/data-generator/lib/tables/products.js +++ b/ghost/data-generator/lib/importers/ProductsImporter.js @@ -1,21 +1,34 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const {blogStartDate} = require('../utils/blog-info'); class ProductsImporter extends TableImporter { static table = 'products'; + static dependencies = []; + defaultQuantity = 4; - constructor(knex) { - super(ProductsImporter.table, knex); + constructor(knex, transaction) { + super(ProductsImporter.table, knex, transaction); } - setImportOptions() { + async import(quantity = this.defaultQuantity) { + // TODO: Add random products if quantity != 4 this.names = ['Free', 'Bronze', 'Silver', 'Gold']; this.count = 0; + + await super.import(quantity); } - async addStripePrices({products, stripeProducts, stripePrices}) { + /** + * Add the stripe products / prices + */ + async finalise() { + const stripeProducts = await this.transaction.select('id', 'product_id', 'stripe_product_id').from('stripe_products'); + const stripePrices = await this.transaction.select('id', 'stripe_product_id', 'interval').from('stripe_prices'); + + const products = await this.transaction.select('id').from('products'); + for (const {id} of products) { const stripeProduct = stripeProducts.find(p => id === p.product_id); if (!stripeProduct) { @@ -40,7 +53,7 @@ class ProductsImporter extends TableImporter { } if (Object.keys(update).length > 0) { - await this.knex('products').update(update).where({ + await this.transaction('products').update(update).where({ id }); } @@ -59,7 +72,7 @@ class ProductsImporter extends TableImporter { }; if (count !== 0) { tierInfo.type = 'paid'; - tierInfo.description = `${name} star member`; + tierInfo.description = `${name} tier member`; tierInfo.currency = 'USD'; tierInfo.monthly_price = count * 500; tierInfo.yearly_price = count * 5000; diff --git a/ghost/data-generator/lib/tables/redirects.js b/ghost/data-generator/lib/importers/RedirectsImporter.js similarity index 55% rename from ghost/data-generator/lib/tables/redirects.js rename to ghost/data-generator/lib/importers/RedirectsImporter.js index 4c93558096..5034bdf9f1 100644 --- a/ghost/data-generator/lib/tables/redirects.js +++ b/ghost/data-generator/lib/importers/RedirectsImporter.js @@ -1,18 +1,32 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); class RedirectsImporter extends TableImporter { static table = 'redirects'; + static dependencies = ['posts']; - constructor(knex) { - super(RedirectsImporter.table, knex); + constructor(knex, transaction) { + super(RedirectsImporter.table, knex, transaction); } - setImportOptions({model, amount}) { + async import(quantity) { + const posts = await this.transaction + .select('id', 'published_at') + .from('posts') + .where('type', 'post') + .andWhere('status', 'published'); + + this.quantity = quantity ? quantity / posts.length : 10; + await this.importForEach(posts, this.quantity); + } + + setReferencedModel(model) { this.model = model; + + // Reset the amount for each model this.amount = faker.datatype.number({ - min: 1, - max: amount + min: 0, + max: this.quantity }); } diff --git a/ghost/data-generator/lib/tables/roles-users.js b/ghost/data-generator/lib/importers/RolesUsersImporter.js similarity index 57% rename from ghost/data-generator/lib/tables/roles-users.js rename to ghost/data-generator/lib/importers/RolesUsersImporter.js index e9d25b42e5..4c830cdf67 100644 --- a/ghost/data-generator/lib/tables/roles-users.js +++ b/ghost/data-generator/lib/importers/RolesUsersImporter.js @@ -1,17 +1,23 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); class RolesUsersImporter extends TableImporter { static table = 'roles_users'; + // No roles imorter, since roles are statically defined in database + static dependencies = ['users']; - constructor(knex, {roles}) { - super(RolesUsersImporter.table, knex); - this.roles = roles; - this.sortOrder = 0; + constructor(knex, transaction) { + super(RolesUsersImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + /** + * Ignore overriden quantity for 1:1 relationship + */ + async import() { + const users = await this.transaction.select('id').from('users').whereNot('id', 1); + this.roles = await this.transaction.select('id', 'name').from('roles'); + + await this.importForEach(users, 1); } generate() { diff --git a/ghost/data-generator/lib/tables/stripe-prices.js b/ghost/data-generator/lib/importers/StripePricesImporter.js similarity index 71% rename from ghost/data-generator/lib/tables/stripe-prices.js rename to ghost/data-generator/lib/importers/StripePricesImporter.js index 546fb6dce7..287531b395 100644 --- a/ghost/data-generator/lib/tables/stripe-prices.js +++ b/ghost/data-generator/lib/importers/StripePricesImporter.js @@ -1,25 +1,31 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {blogStartDate} = require('../utils/blog-info'); +const sixWeeksLater = new Date(blogStartDate); +sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + class StripePricesImporter extends TableImporter { static table = 'stripe_prices'; + static dependencies = ['products', 'stripe_products']; - constructor(knex, {products}) { - super(StripePricesImporter.table, knex); - this.products = products; + constructor(knex, transaction) { + super(StripePricesImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import() { + const stripeProducts = await this.transaction.select('id', 'stripe_product_id', 'product_id').from('stripe_products'); + this.products = await this.transaction.select('id', 'monthly_price', 'yearly_price').from('products'); + await this.importForEach(stripeProducts, 2); + } + + setReferencedModel(model) { + this.model = model; this.count = 0; } generate() { - const sixWeeksLater = new Date(blogStartDate); - sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); - const count = this.count; this.count = this.count + 1; diff --git a/ghost/data-generator/lib/tables/stripe-products.js b/ghost/data-generator/lib/importers/StripeProductsImporter.js similarity index 57% rename from ghost/data-generator/lib/tables/stripe-products.js rename to ghost/data-generator/lib/importers/StripeProductsImporter.js index 055ad2b10d..952336bf1d 100644 --- a/ghost/data-generator/lib/tables/stripe-products.js +++ b/ghost/data-generator/lib/importers/StripeProductsImporter.js @@ -1,20 +1,24 @@ const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {blogStartDate} = require('../utils/blog-info'); +const sixWeeksLater = new Date(blogStartDate); +sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); + class StripeProductsImporter extends TableImporter { static table = 'stripe_products'; - constructor(knex) { - super(StripeProductsImporter.table, knex); + static dependencies = ['products']; + + constructor(knex, transaction) { + super(StripeProductsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import() { + const products = await this.transaction.select('id').from('products'); + await this.importForEach(products, 1); } generate() { - const sixWeeksLater = new Date(blogStartDate); - sixWeeksLater.setDate(sixWeeksLater.getDate() + (7 * 6)); return { id: faker.database.mongodbObjectId(), product_id: this.model.id, diff --git a/ghost/data-generator/lib/tables/subscriptions.js b/ghost/data-generator/lib/importers/SubscriptionsImporter.js similarity index 73% rename from ghost/data-generator/lib/tables/subscriptions.js rename to ghost/data-generator/lib/importers/SubscriptionsImporter.js index 97a9d1aeac..2172fc0833 100644 --- a/ghost/data-generator/lib/tables/subscriptions.js +++ b/ghost/data-generator/lib/importers/SubscriptionsImporter.js @@ -1,20 +1,22 @@ const {faker} = require('@faker-js/faker'); const generateEvents = require('../utils/event-generator'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const dateToDatabaseString = require('../utils/database-date'); class SubscriptionsImporter extends TableImporter { static table = 'subscriptions'; + static dependencies = ['members', 'members_products', 'stripe_products', 'stripe_prices']; - constructor(knex, {members, stripeProducts, stripePrices}) { - super(SubscriptionsImporter.table, knex); - this.members = members; - this.stripeProducts = stripeProducts; - this.stripePrices = stripePrices; + constructor(knex, transaction) { + super(SubscriptionsImporter.table, knex, transaction); } - setImportOptions({model}) { - this.model = model; + async import() { + const membersProducts = await this.transaction.select('member_id', 'product_id').from('members_products'); + this.members = await this.transaction.select('id', 'status', 'created_at').from('members'); + this.stripeProducts = await this.transaction.select('product_id', 'stripe_product_id').from('stripe_products'); + this.stripePrices = await this.transaction.select('stripe_product_id', 'currency', 'amount', 'interval').from('stripe_prices'); + await this.importForEach(membersProducts, 1); } generate() { @@ -28,6 +30,7 @@ class SubscriptionsImporter extends TableImporter { return price.stripe_product_id === stripeProduct.stripe_product_id && (isMonthly ? price.interval === 'month' : price.interval === 'year'); }); + billingInfo.cadence = isMonthly ? 'month' : 'year'; billingInfo.currency = stripePrice.currency; billingInfo.amount = stripePrice.amount; diff --git a/ghost/data-generator/lib/importers/TableImporter.js b/ghost/data-generator/lib/importers/TableImporter.js new file mode 100644 index 0000000000..04f1ac5a56 --- /dev/null +++ b/ghost/data-generator/lib/importers/TableImporter.js @@ -0,0 +1,108 @@ +class TableImporter { + /** + * @type {object|undefined} model Referenced model when generating data + */ + model; + + /** + * @type {number|undefined} defaultQuantity Default number of records to import + */ + defaultQuantity; + + /** + * Transaction and knex need to be separate since we're using the batchInsert helper + * @param {string} name Name of the table to be generated + * @param {import('knex/types').Knex} knex Database connection + * @param {import('knex/types').Knex.Transaction} transaction Transaction to be used for import + */ + constructor(name, knex, transaction) { + this.name = name; + this.knex = knex; + this.transaction = transaction; + } + + async import(amount = this.defaultQuantity) { + const batchSize = 500; + let batch = []; + + for (let i = 0; i < amount; i++) { + const model = await this.generate(); + if (model) { + batch.push(model); + } else { + // After first null assume that there is no more data + break; + } + if (batch.length === batchSize) { + await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction); + batch = []; + } + } + + // Process final batch + if (batch.length > 0) { + await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction); + } + } + + /** + * @param {Array} models List of models to reference + * @param {Number|function} amount Number of records to import per model + */ + async importForEach(models = [], amount) { + const batchSize = 500; + let batch = []; + + for (const model of models) { + this.setReferencedModel(model); + let currentAmount = (typeof amount === 'function') ? amount() : amount; + if (!Number.isInteger(currentAmount)) { + currentAmount = Math.floor(currentAmount) + ((Math.random() < currentAmount % 1) ? 1 : 0); + } + for (let i = 0; i < currentAmount; i++) { + const data = await this.generate(); + if (data) { + batch.push(data); + } else { + // After first null assume that there is no more data for this model + break; + } + if (batch.length === batchSize) { + await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction); + batch = []; + } + } + } + + // Process final batch + if (batch.length > 0) { + await this.knex.batchInsert(this.name, batch, batchSize).transacting(this.transaction); + } + } + + /** + * Finalise the imported data, e.g. adding summary records based on a table's dependents + */ + async finalise() { + // No-op by default + } + + /** + * Sets the model which newly generated data will reference + * @param {Object} model Model to reference when generating data + */ + setReferencedModel(model) { + this.model = model; + } + + /** + * Generates the data for a single model to be imported + * @returns {Object|null} Data to import, optional + */ + generate() { + // Should never be called + return false; + } +} + +module.exports = TableImporter; diff --git a/ghost/data-generator/lib/tables/tags.js b/ghost/data-generator/lib/importers/TagsImporter.js similarity index 70% rename from ghost/data-generator/lib/tables/tags.js rename to ghost/data-generator/lib/importers/TagsImporter.js index 16c2cca6ed..812afd62a1 100644 --- a/ghost/data-generator/lib/tables/tags.js +++ b/ghost/data-generator/lib/importers/TagsImporter.js @@ -1,14 +1,23 @@ const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const dateToDatabaseString = require('../utils/database-date'); class TagsImporter extends TableImporter { static table = 'tags'; + static dependencies = ['users']; + defaultQuantity = faker.datatype.number({ + min: 16, + max: 24 + }); - constructor(knex, {users}) { - super(TagsImporter.table, knex); - this.users = users; + constructor(knex, transaction) { + super(TagsImporter.table, knex, transaction); + } + + async import(quantity = this.defaultQuantity) { + this.users = await this.transaction.select('id').from('users'); + await super.import(quantity); } generate() { diff --git a/ghost/data-generator/lib/tables/users.js b/ghost/data-generator/lib/importers/UsersImporter.js similarity index 81% rename from ghost/data-generator/lib/tables/users.js rename to ghost/data-generator/lib/importers/UsersImporter.js index 516a167e9d..abbe1055c5 100644 --- a/ghost/data-generator/lib/tables/users.js +++ b/ghost/data-generator/lib/importers/UsersImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const {slugify} = require('@tryghost/string'); const security = require('@tryghost/security'); @@ -6,9 +6,11 @@ const dateToDatabaseString = require('../utils/database-date'); class UsersImporter extends TableImporter { static table = 'users'; + static dependencies = []; + defaultQuantity = 8; - constructor(knex) { - super(UsersImporter.table, knex); + constructor(knex, transaction) { + super(UsersImporter.table, knex, transaction); } async generate() { diff --git a/ghost/data-generator/lib/tables/mentions.js b/ghost/data-generator/lib/importers/WebMentionsImporter.js similarity index 56% rename from ghost/data-generator/lib/tables/mentions.js rename to ghost/data-generator/lib/importers/WebMentionsImporter.js index 3781c1842a..4ca62e91bd 100644 --- a/ghost/data-generator/lib/tables/mentions.js +++ b/ghost/data-generator/lib/importers/WebMentionsImporter.js @@ -1,4 +1,4 @@ -const TableImporter = require('./base'); +const TableImporter = require('./TableImporter'); const {faker} = require('@faker-js/faker'); const generateEvents = require('../utils/event-generator'); const {luck} = require('../utils/random'); @@ -6,14 +6,22 @@ const dateToDatabaseString = require('../utils/database-date'); class WebMentionsImporter extends TableImporter { static table = 'mentions'; + static dependencies = ['posts']; - constructor(knex, {baseUrl}) { - super(WebMentionsImporter.table, knex); + constructor(knex, transaction, {baseUrl}) { + super(WebMentionsImporter.table, knex, transaction); this.baseUrl = baseUrl; } - setImportOptions({model, amount}) { + async import(quantity) { + const posts = await this.transaction.select('id', 'slug', 'published_at').from('posts').where('type', 'post'); + + this.quantity = quantity ? quantity / posts.length : 4; + await this.importForEach(posts, this.quantity); + } + + setReferencedModel(model) { this.model = model; // Most web mentions published soon after publication date @@ -23,7 +31,7 @@ class WebMentionsImporter extends TableImporter { this.timestamps = generateEvents({ shape: 'ease-out', trend: 'negative', - total: amount, + total: this.quantity, startTime: startDate, endTime: endDate }).sort(); @@ -31,7 +39,7 @@ class WebMentionsImporter extends TableImporter { generate() { if (luck(50)) { - // 50/50 chance of having a web mention + // 50% chance of 1 mention, 25% chance of 2 mentions, etc. return null; } @@ -39,6 +47,23 @@ class WebMentionsImporter extends TableImporter { const timestamp = this.timestamps.shift(); const author = `${faker.name.fullName()}`; + /** + id: {type: 'string', maxlength: 24, nullable: false, primary: true}, + source: {type: 'string', maxlength: 2000, nullable: false}, + source_title: {type: 'string', maxlength: 2000, nullable: true}, + source_site_title: {type: 'string', maxlength: 2000, nullable: true}, + source_excerpt: {type: 'string', maxlength: 2000, nullable: true}, + source_author: {type: 'string', maxlength: 2000, nullable: true}, + source_featured_image: {type: 'string', maxlength: 2000, nullable: true}, + source_favicon: {type: 'string', maxlength: 2000, nullable: true}, + target: {type: 'string', maxlength: 2000, nullable: false}, + resource_id: {type: 'string', maxlength: 24, nullable: true}, + resource_type: {type: 'string', maxlength: 50, nullable: true}, + created_at: {type: 'dateTime', nullable: false}, + payload: {type: 'text', maxlength: 65535, nullable: true}, + deleted: {type: 'boolean', nullable: false, defaultTo: false}, + verified: {type: 'boolean', nullable: false, defaultTo: false} + */ return { id, source: `${faker.internet.url()}/${faker.helpers.slugify(`${faker.word.adjective()} ${faker.word.noun()}`).toLowerCase()}`, diff --git a/ghost/data-generator/lib/importers/index.js b/ghost/data-generator/lib/importers/index.js new file mode 100644 index 0000000000..4ccbfc32ea --- /dev/null +++ b/ghost/data-generator/lib/importers/index.js @@ -0,0 +1,37 @@ +module.exports = [ + require('./NewslettersImporter'), + require('./PostsImporter'), + require('./UsersImporter'), + require('./TagsImporter'), + require('./ProductsImporter'), + require('./MembersImporter'), + require('./BenefitsImporter'), + require('./WebMentionsImporter'), + require('./PostsAuthorsImporter'), + require('./PostsTagsImporter'), + require('./ProductsBenefitsImporter'), + require('./MembersProductsImporter'), + require('./PostsProductsImporter'), + require('./MembersNewslettersImporter'), + require('./StripeProductsImporter'), + require('./StripePricesImporter'), + require('./SubscriptionsImporter'), + require('./EmailsImporter'), + require('./EmailBatchesImporter'), + require('./EmailRecipientsImporter'), + require('./RedirectsImporter'), + require('./MembersClickEventsImporter'), + require('./OffersImporter'), + require('./MembersCreatedEventsImporter'), + require('./MembersLoginEventsImporter'), + require('./MembersStatusEventsImporter'), + require('./MembersStripeCustomersImporter'), + require('./MembersStripeCustomersSubscriptionsImporter'), + require('./MembersPaidSubscriptionEventsImporter'), + require('./MembersSubscriptionCreatedEventsImporter'), + require('./MembersSubscribeEventsImporter'), + require('./LabelsImporter'), + require('./MembersLabelsImporter'), + require('./RolesUsersImporter'), + require('./MembersFeedbackImporter') +]; diff --git a/ghost/data-generator/lib/tables/base.js b/ghost/data-generator/lib/tables/base.js deleted file mode 100644 index 70574ddc33..0000000000 --- a/ghost/data-generator/lib/tables/base.js +++ /dev/null @@ -1,89 +0,0 @@ -class TableImporter { - /** - * @param {string} name Name of the table to be generated - * @param {import('knex/types').Knex} knex Database connection - */ - constructor(name, knex) { - this.name = name; - this.knex = knex; - } - - /** - * @typedef {Function} AmountFunction - * @returns {number} - */ - - /** - * @typedef {Object.} ImportOptions - * @property {number|AmountFunction} amount Number of events to generate - * @property {Object} [model] Used to reference another object during creation - */ - - /** - * @param {Array} models List of models to reference - * @param {ImportOptions} [options] Import options - * @returns {Promise>} - */ - async importForEach(models = [], options) { - const results = []; - for (const model of models) { - results.push(...await this.import(Object.assign({}, options, {model}))); - } - return results; - } - - /** - * @param {ImportOptions} options Import options - * @returns {Promise>} - */ - async import(options) { - if (options.amount === 0) { - return; - } - - // Use dynamic amount if faker function given - const amount = (typeof options.amount === 'function') ? options.amount() : options.amount; - - this.setImportOptions(Object.assign({}, options, {amount})); - - const data = []; - for (let i = 0; i < amount; i++) { - const model = await this.generate(); - if (model) { - // Only push models when one is generated successfully - data.push(model); - } else { - // After first null assume that there is no more data - break; - } - } - - const rows = ['id']; - if (options && options.rows) { - rows.push(...options.rows); - } - await this.knex.batchInsert(this.name, data, 500); - return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(this.name); - } - - /** - * - * @param {ImportOptions} options - * @returns {void} - */ - // eslint-disable-next-line no-unused-vars - setImportOptions(options) { - return; - } - - /** - * Generates the data for a single model to be imported - * @returns {Object|null} Data to import, optional - */ - generate() { - // Should never be called - return false; - } -} - -module.exports = TableImporter; diff --git a/ghost/data-generator/lib/tables/index.js b/ghost/data-generator/lib/tables/index.js deleted file mode 100644 index 69753ce58e..0000000000 --- a/ghost/data-generator/lib/tables/index.js +++ /dev/null @@ -1,38 +0,0 @@ -// Order matters! Ordered so that dependant tables are after their dependencies -module.exports = { - NewslettersImporter: require('./newsletters'), - PostsImporter: require('./posts'), - UsersImporter: require('./users'), - TagsImporter: require('./tags'), - ProductsImporter: require('./products'), - MembersImporter: require('./members'), - BenefitsImporter: require('./benefits'), - MentionsImporter: require('./mentions'), - PostsAuthorsImporter: require('./posts-authors'), - PostsTagsImporter: require('./posts-tags'), - ProductsBenefitsImporter: require('./products-benefits'), - MembersProductsImporter: require('./members-products'), - PostsProductsImporter: require('./posts-products'), - MembersNewslettersImporter: require('./members-newsletters'), - StripeProductsImporter: require('./stripe-products'), - StripePricesImporter: require('./stripe-prices'), - SubscriptionsImporter: require('./subscriptions'), - EmailsImporter: require('./emails'), - EmailBatchesImporter: require('./email-batches'), - EmailRecipientsImporter: require('./email-recipients'), - RedirectsImporter: require('./redirects'), - MembersClickEventsImporter: require('./members-click-events'), - OffersImporter: require('./offers'), - MembersCreatedEventsImporter: require('./members-created-events'), - MembersLoginEventsImporter: require('./members-login-events'), - MembersStatusEventsImporter: require('./members-status-events'), - MembersStripeCustomersImporter: require('./members-stripe-customers'), - MembersStripeCustomersSubscriptionsImporter: require('./members-stripe-customers-subscriptions'), - MembersPaidSubscriptionEventsImporter: require('./members-paid-subscription-events'), - MembersSubscriptionCreatedEventsImporter: require('./members-subscription-created-events'), - MembersSubscribeEventsImporter: require('./members-subscribe-events'), - LabelsImporter: require('./labels'), - MembersLabelsImporter: require('./members-labels'), - RolesUsersImporter: require('./roles-users'), - MembersFeedbackImporter: require('./members-feedback') -}; diff --git a/ghost/data-generator/lib/tables/members-newsletters.js b/ghost/data-generator/lib/tables/members-newsletters.js deleted file mode 100644 index b35863440c..0000000000 --- a/ghost/data-generator/lib/tables/members-newsletters.js +++ /dev/null @@ -1,24 +0,0 @@ -const {faker} = require('@faker-js/faker'); -const TableImporter = require('./base'); - -class MembersNewslettersImporter extends TableImporter { - static table = 'members_newsletters'; - - constructor(knex) { - super(MembersNewslettersImporter.table, knex); - } - - setImportOptions({model}) { - this.model = model; - } - - generate() { - return { - id: faker.database.mongodbObjectId(), - member_id: this.model.member_id, - newsletter_id: this.model.newsletter_id - }; - } -} - -module.exports = MembersNewslettersImporter; diff --git a/ghost/data-generator/lib/utils/json-importer.js b/ghost/data-generator/lib/utils/JsonImporter.js similarity index 75% rename from ghost/data-generator/lib/utils/json-importer.js rename to ghost/data-generator/lib/utils/JsonImporter.js index c65a269728..ed5f8bd97a 100644 --- a/ghost/data-generator/lib/utils/json-importer.js +++ b/ghost/data-generator/lib/utils/JsonImporter.js @@ -1,8 +1,9 @@ const {faker} = require('@faker-js/faker'); class JsonImporter { - constructor(knex) { + constructor(knex, transaction) { this.knex = knex; + this.transaction = transaction; } /** @@ -15,7 +16,7 @@ class JsonImporter { /** * Import a dataset to the database * @param {JsonImportOptions} options - * @returns {Promise>>} Set of rows returned from database + * @returns {Promise} */ async import({ name, @@ -30,8 +31,7 @@ class JsonImporter { if (rows.findIndex(row => row === 'id') === -1) { rows.unshift('id'); } - await this.knex.batchInsert(name, data, 500); - return await this.knex.select(...rows).whereIn('id', data.map(obj => obj.id)).from(name); + await this.knex.batchInsert(name, data, 500).transacting(this.transaction); } } diff --git a/ghost/data-generator/lib/utils/topological-sort.js b/ghost/data-generator/lib/utils/topological-sort.js new file mode 100644 index 0000000000..972f9e7bde --- /dev/null +++ b/ghost/data-generator/lib/utils/topological-sort.js @@ -0,0 +1,33 @@ +/** + * This sorting algorithm is used to make sure that dependent tables are imported after their dependencies. + * @param {Array} objects Objects with a name and dependencies properties + * @returns Topologically sorted array of objects + */ +module.exports = function topologicalSort(objects) { + // Create an empty result array to store the ordered objects + const result = []; + // Create a set to track visited objects during the DFS + const visited = new Set(); + + // Helper function to perform DFS + function dfs(name) { + if (visited.has(name)) { + return; + } + + visited.add(name); + const dependencies = objects.find(item => item.name === name)?.dependencies || []; + for (const dependency of dependencies) { + dfs(dependency); + } + + result.push(objects.find(item => item.name === name)); + } + + // Perform DFS on each object + for (const object of objects) { + dfs(object.name); + } + + return result; +}; diff --git a/ghost/data-generator/test/data-generator.test.js b/ghost/data-generator/test/data-generator.test.js index b52cfaee1d..a947969437 100644 --- a/ghost/data-generator/test/data-generator.test.js +++ b/ghost/data-generator/test/data-generator.test.js @@ -2,11 +2,10 @@ // const testUtils = require('./utils'); require('./utils'); const knex = require('knex'); -const { - ProductsImporter, - StripeProductsImporter, - StripePricesImporter -} = require('../lib/tables'); +const importers = require('../lib/importers'); +const ProductsImporter = importers.find(i => i.table === 'products'); +const StripeProductsImporter = importers.find(i => i.table === 'stripe_products'); +const StripePricesImporter = importers.find(i => i.table === 'stripe_prices'); const generateEvents = require('../lib/utils/event-generator'); @@ -82,11 +81,14 @@ describe('Data Generator', function () { info: () => { }, ok: () => { } }, - modelQuantities: { - members: 10, - membersLoginEvents: 5, - posts: 2 - } + tables: [{ + name: 'members', + quantity: 10 + }, { + name: 'posts', + quantity: 2 + }], + withDefault: true }); try { return await dataGenerator.importData(); @@ -152,27 +154,25 @@ describe('Importer', function () { }); it('Should import a single item', async function () { - const productsImporter = new ProductsImporter(db); - const products = await productsImporter.import({amount: 1, rows: ['name', 'monthly_price', 'yearly_price']}); + const transaction = await db.transaction(); + const productsImporter = new ProductsImporter(db, transaction); + await productsImporter.import(); + transaction.commit(); - products.length.should.eql(1); + const products = await db.select('id', 'name').from('products'); + + products.length.should.eql(4); products[0].name.should.eql('Free'); - - const results = await db.select('id', 'name').from('products'); - - results.length.should.eql(1); - results[0].name.should.eql('Free'); }); it('Should import an item for each entry in an array', async function () { - const productsImporter = new ProductsImporter(db); - const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + const transaction = await db.transaction(); + const productsImporter = new ProductsImporter(db, transaction); + await productsImporter.import(); - const stripeProductsImporter = new StripeProductsImporter(db); - await stripeProductsImporter.importForEach(products, { - amount: 1, - rows: ['product_id', 'stripe_product_id'] - }); + const stripeProductsImporter = new StripeProductsImporter(db, transaction); + await stripeProductsImporter.import(); + transaction.commit(); const results = await db.select('id').from('stripe_products'); @@ -180,26 +180,20 @@ describe('Importer', function () { }); it('Should update products to reference price ids', async function () { - const productsImporter = new ProductsImporter(db); - const products = await productsImporter.import({amount: 4, rows: ['name', 'monthly_price', 'yearly_price']}); + const transaction = await db.transaction(); + const productsImporter = new ProductsImporter(db, transaction); + await productsImporter.import(); - const stripeProductsImporter = new StripeProductsImporter(db); - const stripeProducts = await stripeProductsImporter.importForEach(products, { - amount: 1, - rows: ['product_id', 'stripe_product_id'] - }); + const stripeProductsImporter = new StripeProductsImporter(db, transaction); + await stripeProductsImporter.import(); - const stripePricesImporter = new StripePricesImporter(db, {products}); - const stripePrices = await stripePricesImporter.importForEach(stripeProducts, { - amount: 2, - rows: ['stripe_price_id', 'interval', 'stripe_product_id', 'currency', 'amount', 'nickname'] - }); + const stripePricesImporter = new StripePricesImporter(db, transaction); + await stripePricesImporter.import(); - await productsImporter.addStripePrices({ - products, - stripeProducts, - stripePrices - }); + await productsImporter.finalise(); + await stripeProductsImporter.finalise(); + await stripePricesImporter.finalise(); + transaction.commit(); const results = await db.select('id', 'name', 'monthly_price_id', 'yearly_price_id').from('products');