From 97d0cddb50c245d849260cf739d0d4e9bba434c9 Mon Sep 17 00:00:00 2001 From: Simon Backx Date: Tue, 31 Oct 2023 16:11:24 +0100 Subject: [PATCH] Added indexes to `members_created_events` and `members_subscription_created_events` (#18805) fixes https://github.com/TryGhost/Product/issues/4085 Increases the performance for the post analytics export by adding new indexes. These indexes are used when counting the amount of (paid) subscribers that were attributed to a given post. With the indexes, the time required to export 700 posts with 300k members decreases from 40s to 0.6s. Tests show that adding these indexes should be very fast (< 1 s) if the tables contain up to 300k rows. --- .../server/data/migrations/utils/schema.js | 19 ++++++- ...00-members-created-attribution-id-index.js | 3 + ...bscription-created-attribution-id-index.js | 3 + .../core/core/server/data/schema/commands.js | 56 +++++++++++++++++++ ghost/core/core/server/data/schema/schema.js | 4 +- .../unit/server/data/schema/integrity.test.js | 2 +- 6 files changed, 83 insertions(+), 4 deletions(-) create mode 100644 ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-00-members-created-attribution-id-index.js create mode 100644 ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-01-members-subscription-created-attribution-id-index.js diff --git a/ghost/core/core/server/data/migrations/utils/schema.js b/ghost/core/core/server/data/migrations/utils/schema.js index 5dee3f7bdf..d72215749a 100644 --- a/ghost/core/core/server/data/migrations/utils/schema.js +++ b/ghost/core/core/server/data/migrations/utils/schema.js @@ -96,6 +96,22 @@ function createSetNullableMigration(table, column, options = {}) { ); } +/** + * @param {string} table + * @param {string[]|string} columns One or multiple columns (in case the index should be for multiple columns) + * @returns {Migration} + */ +function createAddIndexMigration(table, columns) { + return createTransactionalMigration( + async function up(knex) { + await commands.addIndex(table, columns, knex); + }, + async function down(knex) { + await commands.dropIndex(table, columns, knex); + } + ); +} + /** * @param {string} table * @param {string} from @@ -163,7 +179,8 @@ module.exports = { createDropColumnMigration, createSetNullableMigration, createDropNullableMigration, - createRenameColumnMigration + createRenameColumnMigration, + createAddIndexMigration }; /** diff --git a/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-00-members-created-attribution-id-index.js b/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-00-members-created-attribution-id-index.js new file mode 100644 index 0000000000..9bcbb4cc0b --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-00-members-created-attribution-id-index.js @@ -0,0 +1,3 @@ +const {createAddIndexMigration} = require('../../utils'); + +module.exports = createAddIndexMigration('members_created_events', ['attribution_id']); diff --git a/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-01-members-subscription-created-attribution-id-index.js b/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-01-members-subscription-created-attribution-id-index.js new file mode 100644 index 0000000000..346ea18886 --- /dev/null +++ b/ghost/core/core/server/data/migrations/versions/5.72/2023-10-31-11-06-01-members-subscription-created-attribution-id-index.js @@ -0,0 +1,3 @@ +const {createAddIndexMigration} = require('../../utils'); + +module.exports = createAddIndexMigration('members_subscription_created_events', ['attribution_id']); diff --git a/ghost/core/core/server/data/schema/commands.js b/ghost/core/core/server/data/schema/commands.js index c8b804dc93..252225082d 100644 --- a/ghost/core/core/server/data/schema/commands.js +++ b/ghost/core/core/server/data/schema/commands.js @@ -175,6 +175,60 @@ async function renameColumn(tableName, from, to, transaction = db.knex) { }); } +/** + * Adds an non-unique index to a table over the given columns. + * + * @param {string} tableName - name of the table to add indexes to + * @param {string|string[]} columns - column(s) to add indexes for + * @param {import('knex').Knex} [transaction] - connection object containing knex reference + */ +async function addIndex(tableName, columns, transaction = db.knex) { + try { + logging.info(`Adding index for '${columns}' in table '${tableName}'`); + + return await transaction.schema.table(tableName, function (table) { + table.index(columns); + }); + } catch (err) { + if (err.code === 'SQLITE_ERROR') { + logging.warn(`Index for '${columns}' already exists for table '${tableName}'`); + return; + } + if (err.code === 'ER_DUP_KEYNAME') { + logging.warn(`Index for '${columns}' already exists for table '${tableName}'`); + return; + } + throw err; + } +} + +/** + * Drops a non-unique index from a table over the given columns. + * + * @param {string} tableName - name of the table to remove indexes from + * @param {string|string[]} columns - column(s) to remove indexes for + * @param {import('knex').Knex} [transaction] - connection object containing knex reference + */ +async function dropIndex(tableName, columns, transaction = db.knex) { + try { + logging.info(`Dropping index for '${columns}' in table '${tableName}'`); + + return await transaction.schema.table(tableName, function (table) { + table.dropIndex(columns); + }); + } catch (err) { + if (err.code === 'SQLITE_ERROR') { + logging.warn(`Constraint for '${columns}' does not exist for table '${tableName}'`); + return; + } + if (err.code === 'ER_CANT_DROP_FIELD_OR_KEY') { + logging.warn(`Constraint for '${columns}' does not exist for table '${tableName}'`); + return; + } + throw err; + } +} + /** * Adds an unique index to a table over the given columns. * @@ -535,6 +589,8 @@ module.exports = { getIndexes, addUnique, dropUnique, + addIndex, + dropIndex, addPrimaryKey, addForeign, dropForeign, diff --git a/ghost/core/core/server/data/schema/schema.js b/ghost/core/core/server/data/schema/schema.js index 29a748ecac..2fbe469973 100644 --- a/ghost/core/core/server/data/schema/schema.js +++ b/ghost/core/core/server/data/schema/schema.js @@ -523,7 +523,7 @@ module.exports = { id: {type: 'string', maxlength: 24, nullable: false, primary: true}, created_at: {type: 'dateTime', nullable: false}, member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true}, - attribution_id: {type: 'string', maxlength: 24, nullable: true}, + attribution_id: {type: 'string', maxlength: 24, nullable: true, index: true}, attribution_type: { type: 'string', maxlength: 50, nullable: true, validations: { isIn: [['url', 'post', 'page', 'author', 'tag']] @@ -709,7 +709,7 @@ module.exports = { created_at: {type: 'dateTime', nullable: false}, member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true}, subscription_id: {type: 'string', maxlength: 24, nullable: false, references: 'members_stripe_customers_subscriptions.id', cascadeDelete: true}, - attribution_id: {type: 'string', maxlength: 24, nullable: true}, + attribution_id: {type: 'string', maxlength: 24, nullable: true, index: true}, attribution_type: { type: 'string', maxlength: 50, nullable: true, validations: { isIn: [['url', 'post', 'page', 'author', 'tag']] diff --git a/ghost/core/test/unit/server/data/schema/integrity.test.js b/ghost/core/test/unit/server/data/schema/integrity.test.js index 238935aa98..bf94233d48 100644 --- a/ghost/core/test/unit/server/data/schema/integrity.test.js +++ b/ghost/core/test/unit/server/data/schema/integrity.test.js @@ -35,7 +35,7 @@ const validateRouteSettings = require('../../../../../core/server/services/route */ describe('DB version integrity', function () { // Only these variables should need updating - const currentSchemaHash = '013334f4f51aae785b9afd345861c06a'; + const currentSchemaHash = '1c1f476e830ce01a0167229697bd4523'; const currentFixturesHash = '4db87173699ad9c9d8a67ccab96dfd2d'; const currentSettingsHash = '3128d4ec667a50049486b0c21f04be07'; const currentRoutesHash = '3d180d52c663d173a6be791ef411ed01';