From 77819d261a102e057339e3115565e079234cdba2 Mon Sep 17 00:00:00 2001 From: "Fabien \"egg\" O'Carroll" Date: Thu, 21 Apr 2022 13:10:33 +0100 Subject: [PATCH] Moved Stats Service from Ghost The functionality hasn't changed this has just moved the code and updated the tests to provide better coverage --- ghost/stats-service/.eslintrc.js | 6 + ghost/stats-service/LICENSE | 21 + ghost/stats-service/README.md | 41 ++ ghost/stats-service/index.js | 1 + ghost/stats-service/lib/members.js | 165 ++++++++ ghost/stats-service/lib/mrr.js | 153 ++++++++ ghost/stats-service/lib/stats.js | 37 ++ ghost/stats-service/package.json | 38 ++ ghost/stats-service/test/.eslintrc.js | 6 + ghost/stats-service/test/lib/members.test.js | 373 ++++++++++++++++++ ghost/stats-service/test/lib/mrr.test.js | 386 +++++++++++++++++++ ghost/stats-service/test/lib/stats.test.js | 10 + ghost/stats-service/tsconfig.json | 101 +++++ 13 files changed, 1338 insertions(+) create mode 100644 ghost/stats-service/.eslintrc.js create mode 100644 ghost/stats-service/LICENSE create mode 100644 ghost/stats-service/README.md create mode 100644 ghost/stats-service/index.js create mode 100644 ghost/stats-service/lib/members.js create mode 100644 ghost/stats-service/lib/mrr.js create mode 100644 ghost/stats-service/lib/stats.js create mode 100644 ghost/stats-service/package.json create mode 100644 ghost/stats-service/test/.eslintrc.js create mode 100644 ghost/stats-service/test/lib/members.test.js create mode 100644 ghost/stats-service/test/lib/mrr.test.js create mode 100644 ghost/stats-service/test/lib/stats.test.js create mode 100644 ghost/stats-service/tsconfig.json diff --git a/ghost/stats-service/.eslintrc.js b/ghost/stats-service/.eslintrc.js new file mode 100644 index 0000000000..c9c1bcb522 --- /dev/null +++ b/ghost/stats-service/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/node' + ] +}; diff --git a/ghost/stats-service/LICENSE b/ghost/stats-service/LICENSE new file mode 100644 index 0000000000..19bcb01bef --- /dev/null +++ b/ghost/stats-service/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2013-2022 Ghost Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/ghost/stats-service/README.md b/ghost/stats-service/README.md new file mode 100644 index 0000000000..186bcb8ca8 --- /dev/null +++ b/ghost/stats-service/README.md @@ -0,0 +1,41 @@ +# Stats + +Stats service + +## Install + +`npm install @tryghost/stats --save` + +or + +`yarn add @tryghost/stats` + + +## Usage + + +## Develop + +This is a mono repository, managed with [lerna](https://lernajs.io/). + +Follow the instructions for the top-level repo. +1. `git clone` this repo & `cd` into it as usual +2. Run `yarn` to install top-level dependencies. + + +## Run + +- `yarn dev` + + +## Test + +- `yarn lint` run just eslint +- `yarn test` run lint and tests + + + + +# Copyright & License + +Copyright (c) 2013-2022 Ghost Foundation - Released under the [MIT license](LICENSE). \ No newline at end of file diff --git a/ghost/stats-service/index.js b/ghost/stats-service/index.js new file mode 100644 index 0000000000..61a8b27222 --- /dev/null +++ b/ghost/stats-service/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/stats'); diff --git a/ghost/stats-service/lib/members.js b/ghost/stats-service/lib/members.js new file mode 100644 index 0000000000..ce46b16e04 --- /dev/null +++ b/ghost/stats-service/lib/members.js @@ -0,0 +1,165 @@ +const moment = require('moment'); + +class MembersStatsService { + /** + * @param {object} deps + * @param {import('knex').Knex} deps.knex*/ + constructor({knex}) { + this.knex = knex; + } + + /** + * Get the current total members grouped by status + * @returns {Promise} + */ + async getCount() { + const knex = this.knex; + const rows = await knex('members') + .select('status') + .select(knex.raw('COUNT(id) AS total')) + .groupBy('status'); + + const paidEvent = rows.find(c => c.status === 'paid'); + const freeEvent = rows.find(c => c.status === 'free'); + const compedEvent = rows.find(c => c.status === 'comped'); + + return { + paid: paidEvent ? paidEvent.total : 0, + free: freeEvent ? freeEvent.total : 0, + comped: compedEvent ? compedEvent.total : 0 + }; + } + + /** + * Get the member deltas by status for all days, sorted ascending + * @returns {Promise} The deltas of paid, free and comped users per day, sorted ascending + */ + async fetchAllStatusDeltas() { + const knex = this.knex; + const rows = await knex('members_status_events') + .select(knex.raw('DATE(created_at) as date')) + .select(knex.raw(`SUM( + CASE WHEN to_status='paid' THEN 1 + ELSE 0 END + ) as paid_subscribed`)) + .select(knex.raw(`SUM( + CASE WHEN from_status='paid' THEN 1 + ELSE 0 END + ) as paid_canceled`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='comped' THEN 1 + WHEN from_status='comped' THEN -1 + ELSE 0 END + ) as comped_delta`)) + .select(knex.raw(`SUM( + CASE WHEN to_status='free' THEN 1 + WHEN from_status='free' THEN -1 + ELSE 0 END + ) as free_delta`)) + .groupByRaw('DATE(created_at)') + .orderByRaw('DATE(created_at)'); + return rows; + } + + /** + * Returns a list of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled + * @returns {Promise} + */ + async getCountHistory() { + const rows = await this.fetchAllStatusDeltas(); + + // Fetch current total amounts and start counting from there + const totals = await this.getCount(); + let {paid, free, comped} = totals; + + // Get today in UTC (default timezone) + const today = moment().format('YYYY-MM-DD'); + + const cumulativeResults = []; + + // Loop in reverse order (needed to have correct sorted result) + for (let i = rows.length - 1; i >= 0; i -= 1) { + const row = rows[i]; + + // Convert JSDates to YYYY-MM-DD (in UTC) + const date = moment(row.date).format('YYYY-MM-DD'); + if (date > today) { + // Skip results that are in the future (fix for invalid events) + continue; + } + cumulativeResults.unshift({ + date, + paid: Math.max(0, paid), + free: Math.max(0, free), + comped: Math.max(0, comped), + + // Deltas + paid_subscribed: row.paid_subscribed, + paid_canceled: row.paid_canceled + }); + + // Update current counts + paid -= row.paid_subscribed - row.paid_canceled; + free -= row.free_delta; + comped -= row.comped_delta; + } + + // Now also add the oldest day we have left over (this one will be zero, which is also needed as a data point for graphs) + const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today; + + cumulativeResults.unshift({ + date: oldestDate, + paid: Math.max(0, paid), + free: Math.max(0, free), + comped: Math.max(0, comped), + + // Deltas + paid_subscribed: 0, + paid_canceled: 0 + }); + + return { + data: cumulativeResults, + meta: { + totals + } + }; + } +} + +module.exports = MembersStatsService; + +/** + * @typedef MemberStatusDelta + * @type {Object} + * @property {Date} date + * @property {number} paid_subscribed Paid members that subscribed on this day + * @property {number} paid_canceled Paid members that canceled on this day + * @property {number} comped_delta Total net comped members on this day + * @property {number} free_delta Total net members on this day + */ + +/** + * @typedef TotalMembersByStatus + * @type {Object} + * @property {number} paid Total paid members + * @property {number} free Total free members + * @property {number} comped Total comped members + */ + +/** + * @typedef {Object} TotalMembersByStatusItem + * @property {string} date In YYYY-MM-DD format + * @property {number} paid Total paid members + * @property {number} free Total free members + * @property {number} comped Total comped members + * @property {number} paid_subscribed Paid members that subscribed on this day + * @property {number} paid_canceled Paid members that canceled on this day + */ + +/** + * @typedef {Object} CountHistory + * @property {TotalMembersByStatusItem[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled + * @property {Object} meta + * @property {TotalMembersByStatus} meta.totals + */ diff --git a/ghost/stats-service/lib/mrr.js b/ghost/stats-service/lib/mrr.js new file mode 100644 index 0000000000..29d7112a1e --- /dev/null +++ b/ghost/stats-service/lib/mrr.js @@ -0,0 +1,153 @@ +const moment = require('moment'); + +class MrrStatsService { + /** + * @param {object} deps + * @param {import('knex').Knex} deps.knex + **/ + constructor({knex}) { + this.knex = knex; + } + + /** + * Get the current total MRR, grouped by currency (ascending order) + * @returns {Promise} + */ + async getCurrentMrr() { + const knex = this.knex; + const rows = await knex('members_stripe_customers_subscriptions') + .select(knex.raw(`plan_currency as currency`)) + .select(knex.raw(`SUM(mrr) AS mrr`)) + .groupBy('plan_currency') + .orderBy('currency'); + + if (rows.length === 0) { + // Add a USD placeholder to always have at least one currency + rows.push({ + currency: 'usd', + mrr: 0 + }); + } + + return rows; + } + + /** + * Get the MRR deltas for all days (from old to new), grouped by currency (ascending alphabetically) + * @returns {Promise} The deltas sorted from new to old + */ + async fetchAllDeltas() { + const knex = this.knex; + const rows = await knex('members_paid_subscription_events') + .select('currency') + // In SQLite, DATE(created_at) would map to a string value, while DATE(created_at) would map to a JSDate object in MySQL + // That is why we need the cast here (to have some consistency) + .select(knex.raw('CAST(DATE(created_at) as CHAR) as date')) + .select(knex.raw(`SUM(mrr_delta) as delta`)) + .groupByRaw('CAST(DATE(created_at) as CHAR), currency') + .orderByRaw('CAST(DATE(created_at) as CHAR), currency'); + return rows; + } + + /** + * Returns a list of the MRR history for each day and currency, including the current MRR per currency as meta data. + * The respons is in ascending date order, and currencies for the same date are always in ascending order. + * @returns {Promise} + */ + async getHistory() { + // Fetch current total amounts and start counting from there + const totals = await this.getCurrentMrr(); + + const rows = await this.fetchAllDeltas(); + + // Get today in UTC (default timezone) + const today = moment().format('YYYY-MM-DD'); + + const results = []; + + // Create a map of the totals by currency for fast lookup and editing + + /** @type {Object.}*/ + const currentTotals = {}; + for (const total of totals) { + currentTotals[total.currency] = total.mrr; + } + + // Loop in reverse order (needed to have correct sorted result) + for (let i = rows.length - 1; i >= 0; i -= 1) { + const row = rows[i]; + + if (currentTotals[row.currency] === undefined) { + // Skip unexpected currencies that are not in the totals + continue; + } + + // Convert JSDates to YYYY-MM-DD (in UTC) + const date = moment(row.date).format('YYYY-MM-DD'); + + if (date > today) { + // Skip results that are in the future for some reason + continue; + } + + results.unshift({ + date, + mrr: Math.max(0, currentTotals[row.currency]), + currency: row.currency + }); + + currentTotals[row.currency] -= row.delta; + } + + // Now also add the oldest days we have left over and do not have deltas + const oldestDate = rows.length > 0 ? moment(rows[0].date).add(-1, 'days').format('YYYY-MM-DD') : today; + + // Note that we also need to loop the totals in reverse order because we need to unshift + for (let i = totals.length - 1; i >= 0; i -= 1) { + const total = totals[i]; + results.unshift({ + date: oldestDate, + mrr: Math.max(0, currentTotals[total.currency]), + currency: total.currency + }); + } + + return { + data: results, + meta: { + totals + } + }; + } +} + +module.exports = MrrStatsService; + +/** + * @typedef MrrByCurrency + * @type {Object} + * @property {number} mrr + * @property {string} currency + */ + +/** + * @typedef MrrDelta + * @type {Object} + * @property {Date} date + * @property {string} currency + * @property {number} delta MRR change on this day + */ + +/** + * @typedef {Object} MrrRecord + * @property {string} date In YYYY-MM-DD format + * @property {string} currency + * @property {number} mrr MRR on this day + */ + +/** + * @typedef {Object} MrrHistory + * @property {MrrRecord[]} data List of the total members by status for each day, including the paid deltas paid_subscribed and paid_canceled + * @property {Object} meta + * @property {MrrByCurrency[]} meta.totals + */ diff --git a/ghost/stats-service/lib/stats.js b/ghost/stats-service/lib/stats.js new file mode 100644 index 0000000000..09879e56e4 --- /dev/null +++ b/ghost/stats-service/lib/stats.js @@ -0,0 +1,37 @@ +const MRRService = require('./mrr'); +const MembersService = require('./members'); + +class StatsService { + /** + * @param {object} deps + * @param {MRRService} deps.mrr + * @param {MembersService} deps.members + **/ + constructor(deps) { + this.mrr = deps.mrr; + this.members = deps.members; + } + + async getMRRHistory() { + return this.mrr.getHistory(); + } + + async getMemberCountHistory() { + return this.members.getCountHistory(); + } + + /** + * @param {object} deps + * @param {import('knex').Knex} deps.knex + * + * @returns {StatsService} + **/ + static create(deps) { + return new StatsService({ + mrr: new MRRService(deps), + members: new MembersService(deps) + }); + } +} + +module.exports = StatsService; diff --git a/ghost/stats-service/package.json b/ghost/stats-service/package.json new file mode 100644 index 0000000000..f381f5a0bd --- /dev/null +++ b/ghost/stats-service/package.json @@ -0,0 +1,38 @@ +{ + "name": "@tryghost/stats", + "version": "0.0.0", + "repository": "https://github.com/TryGhost/Analytics/tree/main/packages/stats", + "author": "Ghost Foundation", + "license": "MIT", + "main": "index.js", + "scripts": { + "dev": "echo \"Implement me!\"", + "test": "NODE_ENV=testing c8 --all --check-coverage --reporter text --reporter cobertura mocha './test/**/*.test.js'", + "lint:code": "eslint *.js lib/ --ext .js --cache", + "lint": "yarn lint:code && yarn lint:test", + "lint:test": "eslint -c test/.eslintrc.js test/ --ext .js --cache", + "posttest": "yarn lint" + }, + "files": [ + "index.js", + "lib" + ], + "publishConfig": { + "access": "public" + }, + "devDependencies": { + "@types/mocha": "^9.1.0", + "@types/node": "^17.0.25", + "@types/sinon": "^10.0.11", + "@vscode/sqlite3": "^5.0.8", + "c8": "7.11.2", + "knex": "^1.0.7", + "mocha": "9.2.2", + "should": "13.2.3", + "sinon": "13.0.2", + "typescript": "^4.6.3" + }, + "dependencies": { + "moment": "^2.29.3" + } +} diff --git a/ghost/stats-service/test/.eslintrc.js b/ghost/stats-service/test/.eslintrc.js new file mode 100644 index 0000000000..829b601eb0 --- /dev/null +++ b/ghost/stats-service/test/.eslintrc.js @@ -0,0 +1,6 @@ +module.exports = { + plugins: ['ghost'], + extends: [ + 'plugin:ghost/test' + ] +}; diff --git a/ghost/stats-service/test/lib/members.test.js b/ghost/stats-service/test/lib/members.test.js new file mode 100644 index 0000000000..2b5f9d42a4 --- /dev/null +++ b/ghost/stats-service/test/lib/members.test.js @@ -0,0 +1,373 @@ +const MembersStatsService = require('../../lib/members'); +const knex = require('knex').default; +const assert = require('assert'); +const moment = require('moment'); +const sinon = require('sinon'); + +describe('MembersStatsService', function () { + describe('getCountHistory', function () { + /** @type {MembersStatsService} */ + let membersStatsService; + + /** + * @type {MembersStatsService.TotalMembersByStatus} + */ + const currentCounts = {paid: 0, free: 0, comped: 0}; + /** + * @type {MembersStatsService.MemberStatusDelta[]} + */ + let events = []; + + const today = '2000-01-10'; + const tomorrow = '2000-01-11'; + const yesterday = '2000-01-09'; + const dayBeforeYesterday = '2000-01-08'; + const twoDaysBeforeYesterday = '2000-01-07'; + + /** @type {Date} */ + let todayDate; + /** @type {Date} */ + let tomorrowDate; + /** @type {Date} */ + let yesterdayDate; + /** @type {Date} */ + let dayBeforeYesterdayDate; + + after(function () { + sinon.restore(); + }); + + /** @type {import('knex').Knex} */ + let db; + + before(function () { + todayDate = moment(today).toDate(); + tomorrowDate = moment(tomorrow).toDate(); + yesterdayDate = moment(yesterday).toDate(); + dayBeforeYesterdayDate = moment(dayBeforeYesterday).toDate(); + sinon.useFakeTimers(todayDate.getTime()); + membersStatsService = new MembersStatsService({knex: knex({client: 'sqlite3', connection: {filename: ':memory:'}})}); + }); + + beforeEach(async function () { + db = knex({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true}); + membersStatsService = new MembersStatsService({knex: db}); + + await db.schema.createTable('members_status_events', function (table) { + table.string('from_status'); + table.string('to_status'); + table.date('created_at'); + }); + + await db.schema.createTable('members', function (table) { + table.string('id'); + table.string('status'); + }); + }); + + async function setupDB() { + const paidMembers = Array.from({length: currentCounts.paid}).map(() => ({ + id: 'id', + status: 'paid' + })); + const freeMembers = Array.from({length: currentCounts.free}).map(() => ({ + id: 'id', + status: 'free' + })); + const compedMembers = Array.from({length: currentCounts.comped}).map(() => ({ + id: 'id', + status: 'comped' + })); + + await db('members').insert(paidMembers.concat(freeMembers, compedMembers)); + + /** + * @typedef {object} StatusEvent + * @prop {string} created_at + * @prop {string|null} from_status + * @prop {string|null} to_status + **/ + + /** + * @param {string} status + * @param {number} number + * @param {Date} date + * @returns {StatusEvent[]} + **/ + function generateEvents(status, number, date) { + return Array.from({length: Math.abs(number)}).map(() => ({ + created_at: date.toISOString(), + from_status: number > 0 ? null : status, + to_status: number < 0 ? null : status + })); + } + + const toInsert = events.reduce((/** @type {StatusEvent[]} */memo, event) => { + const paidSubscribed = generateEvents('paid', event.paid_subscribed, event.date); + const paidCanceled = generateEvents('paid', -event.paid_canceled, event.date); + const freeSubscribed = generateEvents('free', event.free_delta, event.date); + const compedSubscribed = generateEvents('comped', event.comped_delta, event.date); + return memo.concat(paidSubscribed, paidCanceled, freeSubscribed, compedSubscribed); + }, []); + + if (toInsert.length) { + await db('members_status_events').insert(toInsert); + } + } + + it('Always returns at least one value', async function () { + // No status events + events = []; + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + await setupDB(); + + const {data: results, meta} = await membersStatsService.getCountHistory(); + assert(results.length === 1, 'Should have one result'); + assert.deepEqual(results[0], { + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 0, + paid_canceled: 0 + }); + assert.deepEqual(meta.totals, currentCounts); + }); + + it('Passes paid_subscribers and paid_canceled', async function () { + // Update faked status events + events = [ + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + } + ]; + + // Update current faked counts + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + await setupDB(); + + const {data: results, meta} = await membersStatsService.getCountHistory(); + assert.deepEqual(results, [ + { + date: yesterday, + paid: 0, + free: 0, + comped: 0, + paid_subscribed: 0, + paid_canceled: 0 + }, + { + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + assert.deepEqual(meta.totals, currentCounts); + }); + + it('Correctly resolves deltas', async function () { + // Update faked status events + events = [ + { + date: yesterdayDate, + paid_subscribed: 2, + paid_canceled: 1, + free_delta: 0, + comped_delta: 0 + }, + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + } + ]; + + // Update current faked counts + currentCounts.paid = 2; + currentCounts.free = 3; + currentCounts.comped = 4; + + await setupDB(); + + const {data: results, meta} = await membersStatsService.getCountHistory(); + assert.deepEqual(results, [ + { + date: dayBeforeYesterday, + paid: 0, + free: 1, + comped: 1, + paid_subscribed: 0, + paid_canceled: 0 + }, + { + date: yesterday, + paid: 1, + free: 1, + comped: 1, + paid_subscribed: 2, + paid_canceled: 1 + }, + { + date: today, + paid: 2, + free: 3, + comped: 4, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + assert.deepEqual(meta.totals, currentCounts); + }); + + it('Correctly handles negative numbers', async function () { + // Update faked status events + events = [ + { + date: dayBeforeYesterdayDate, + paid_subscribed: 2, + paid_canceled: 1, + free_delta: 2, + comped_delta: 10 + }, + { + date: yesterdayDate, + paid_subscribed: 2, + paid_canceled: 1, + free_delta: -100, + comped_delta: 0 + }, + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 100, + comped_delta: 3 + } + ]; + + // Update current faked counts + currentCounts.paid = 2; + currentCounts.free = 3; + currentCounts.comped = 4; + + await setupDB(); + + const {data: results, meta} = await membersStatsService.getCountHistory(); + assert.deepEqual(results, [ + { + date: twoDaysBeforeYesterday, + paid: 0, + free: 1, + comped: 0, + paid_subscribed: 0, + paid_canceled: 0 + }, + { + date: dayBeforeYesterday, + paid: 0, + // note that this shouldn't be 100 (which is also what we test here): + free: 3, + comped: 1, + paid_subscribed: 2, + paid_canceled: 1 + }, + { + date: yesterday, + paid: 1, + // never return negative numbers, this is in fact -997: + free: 0, + comped: 1, + paid_subscribed: 2, + paid_canceled: 1 + }, + { + date: today, + paid: 2, + free: 3, + comped: 4, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + assert.deepEqual(meta.totals, currentCounts); + }); + + it('Ignores events in the future', async function () { + // Update faked status events + events = [ + { + date: yesterdayDate, + paid_subscribed: 1, + paid_canceled: 0, + free_delta: 1, + comped_delta: 0 + }, + { + date: todayDate, + paid_subscribed: 4, + paid_canceled: 3, + free_delta: 2, + comped_delta: 3 + }, + { + date: tomorrowDate, + paid_subscribed: 10, + paid_canceled: 5, + free_delta: 8, + comped_delta: 9 + } + ]; + + // Update current faked counts + currentCounts.paid = 1; + currentCounts.free = 2; + currentCounts.comped = 3; + + await setupDB(); + + const {data: results, meta} = await membersStatsService.getCountHistory(); + assert.deepEqual(results, [ + { + date: dayBeforeYesterday, + paid: 0, + free: 0, + comped: 0, + paid_subscribed: 0, + paid_canceled: 0 + }, + { + date: yesterday, + paid: 0, + free: 0, + comped: 0, + paid_subscribed: 1, + paid_canceled: 0 + }, + { + date: today, + paid: 1, + free: 2, + comped: 3, + paid_subscribed: 4, + paid_canceled: 3 + } + ]); + assert.deepEqual(meta.totals, currentCounts); + }); + }); +}); diff --git a/ghost/stats-service/test/lib/mrr.test.js b/ghost/stats-service/test/lib/mrr.test.js new file mode 100644 index 0000000000..d62c843f77 --- /dev/null +++ b/ghost/stats-service/test/lib/mrr.test.js @@ -0,0 +1,386 @@ +const MrrStatsService = require('../../lib/mrr'); +const moment = require('moment'); +const sinon = require('sinon'); +const knex = require('knex').default; +require('should'); + +describe('MrrStatsService', function () { + describe('getHistory', function () { + /** @type {MrrStatsService} */ + let mrrStatsService; + + const today = '2000-01-10'; + const tomorrow = '2000-01-11'; + const yesterday = '2000-01-09'; + const dayBeforeYesterday = '2000-01-08'; + const twoDaysBeforeYesterday = '2000-01-07'; + + after(function () { + sinon.restore(); + }); + + /** @type {import('knex').Knex} */ + let db; + + before(function () { + const todayDate = moment(today).toDate(); + sinon.useFakeTimers(todayDate.getTime()); + }); + + beforeEach(async function () { + db = knex({client: 'sqlite3', connection: {filename: ':memory:'}, useNullAsDefault: true}); + mrrStatsService = new MrrStatsService({knex: db}); + + await db.schema.createTable('members_paid_subscription_events', function (table) { + table.string('currency'); + table.string('mrr_delta'); + table.date('created_at'); + }); + + await db.schema.createTable('members_stripe_customers_subscriptions', function (table) { + table.string('plan_currency'); + table.string('mrr'); + }); + }); + + it('Handles no data', async function () { + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(1); + + // Note that currencies should always be sorted ascending, so EUR should be first. + results[0].should.eql({ + date: today, + mrr: 0, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 0, + currency: 'usd' + } + ]); + }); + + it('Always returns at least one value', async function () { + await db('members_stripe_customers_subscriptions').insert([{ + plan_currency: 'usd', + mrr: 1 + }, { + plan_currency: 'eur', + mrr: 2 + }]); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(2); + + // Note that currencies should always be sorted ascending, so EUR should be first. + results[0].should.eql({ + date: today, + mrr: 2, + currency: 'eur' + }); + results[1].should.eql({ + date: today, + mrr: 1, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 2, + currency: 'eur' + }, + { + mrr: 1, + currency: 'usd' + } + ]); + }); + + it('Does not substract delta of first event', async function () { + await db('members_stripe_customers_subscriptions').insert([{ + plan_currency: 'usd', + mrr: 5 + }]); + await db('members_paid_subscription_events').insert([{ + created_at: today, + mrr_delta: 5, + currency: 'usd' + }]); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(2); + results[0].should.eql({ + date: yesterday, + mrr: 0, + currency: 'usd' + }); + results[1].should.eql({ + date: today, + mrr: 5, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 5, + currency: 'usd' + } + ]); + }); + + it('Correctly calculates deltas', async function () { + await db('members_paid_subscription_events').insert([{ + created_at: yesterday, + mrr_delta: 2, + currency: 'usd' + }, + { + created_at: today, + mrr_delta: 5, + currency: 'usd' + }]); + + await db('members_stripe_customers_subscriptions').insert([{ + plan_currency: 'usd', + mrr: 2 + }, { + plan_currency: 'usd', + mrr: 5 + }]); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(3); + results[0].should.eql({ + date: dayBeforeYesterday, + mrr: 0, + currency: 'usd' + }); + results[1].should.eql({ + date: yesterday, + mrr: 2, + currency: 'usd' + }); + results[2].should.eql({ + date: today, + mrr: 7, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 7, + currency: 'usd' + } + ]); + }); + + it('Correctly calculates deltas for multiple currencies', async function () { + await db('members_paid_subscription_events').insert([ + { + created_at: yesterday, + mrr_delta: 200, + currency: 'eur' + }, + { + created_at: yesterday, + mrr_delta: 2, + currency: 'usd' + }, + { + created_at: today, + mrr_delta: 800, + currency: 'eur' + }, + { + created_at: today, + mrr_delta: 5, + currency: 'usd' + } + ]); + + await db('members_stripe_customers_subscriptions').insert([{ + plan_currency: 'eur', + mrr: 200 + }, { + plan_currency: 'usd', + mrr: 2 + }, { + plan_currency: 'eur', + mrr: 800 + }, { + plan_currency: 'usd', + mrr: 5 + }, { + plan_currency: 'eur', + mrr: 200 + }]); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(6); + results[0].should.eql({ + date: dayBeforeYesterday, + mrr: 200, + currency: 'eur' + }); + results[1].should.eql({ + date: dayBeforeYesterday, + mrr: 0, + currency: 'usd' + }); + results[2].should.eql({ + date: yesterday, + mrr: 400, + currency: 'eur' + }); + results[3].should.eql({ + date: yesterday, + mrr: 2, + currency: 'usd' + }); + results[4].should.eql({ + date: today, + mrr: 1200, + currency: 'eur' + }); + results[5].should.eql({ + date: today, + mrr: 7, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 1200, + currency: 'eur' + }, + { + mrr: 7, + currency: 'usd' + } + ]); + }); + + it('Ignores invalid currencies in deltas', async function () { + await db('members_paid_subscription_events').insert({ + created_at: today, + mrr_delta: 200, + currency: 'abc' + }); + + await db('members_stripe_customers_subscriptions').insert({ + plan_currency: 'usd', + mrr: 7 + }); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(1); + results[0].should.eql({ + date: yesterday, + mrr: 7, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 7, + currency: 'usd' + } + ]); + }); + + it('Ignores events in the future', async function () { + await db('members_paid_subscription_events').insert([ + { + created_at: yesterday, + mrr_delta: 2, + currency: 'usd' + }, + { + created_at: today, + mrr_delta: 5, + currency: 'usd' + }, + { + created_at: tomorrow, + mrr_delta: 10, + currency: 'usd' + } + ]); + + await db('members_stripe_customers_subscriptions').insert({plan_currency: 'usd', mrr: 7}); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(3); + results[0].should.eql({ + date: dayBeforeYesterday, + mrr: 0, + currency: 'usd' + }); + results[1].should.eql({ + date: yesterday, + mrr: 2, + currency: 'usd' + }); + results[2].should.eql({ + date: today, + mrr: 7, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 7, + currency: 'usd' + } + ]); + }); + + it('Correctly handles negative total MRR', async function () { + await db('members_paid_subscription_events').insert([ + { + created_at: dayBeforeYesterday, + mrr_delta: 2, + currency: 'usd' + }, + { + created_at: yesterday, + mrr_delta: -1000, + currency: 'usd' + }, + { + created_at: today, + mrr_delta: 1000, + currency: 'usd' + } + ]); + + await db('members_stripe_customers_subscriptions').insert({plan_currency: 'usd', mrr: 7}); + + const {data: results, meta} = await mrrStatsService.getHistory(); + results.length.should.eql(4); + results[0].should.eql({ + date: twoDaysBeforeYesterday, + mrr: 5, + currency: 'usd' + }); + results[1].should.eql({ + date: dayBeforeYesterday, + // We are mainly testing that this should not be 1000! + mrr: 7, + currency: 'usd' + }); + results[2].should.eql({ + date: yesterday, + // Should never be shown negative (in fact it is -993 here) + mrr: 0, + currency: 'usd' + }); + results[3].should.eql({ + date: today, + mrr: 7, + currency: 'usd' + }); + meta.totals.should.eql([ + { + mrr: 7, + currency: 'usd' + } + ]); + }); + }); +}); diff --git a/ghost/stats-service/test/lib/stats.test.js b/ghost/stats-service/test/lib/stats.test.js new file mode 100644 index 0000000000..9a5acc063b --- /dev/null +++ b/ghost/stats-service/test/lib/stats.test.js @@ -0,0 +1,10 @@ +const StatsService = require('../../lib/stats'); +const knex = require('knex').default; +const assert = require('assert'); + +describe('StatsService', function () { + it('Exposes a create factory', function () { + const service = StatsService.create({knex: knex({client: 'sqlite3', connection: {filename: ':memory:'}})}); + assert(service instanceof StatsService); + }); +}); diff --git a/ghost/stats-service/tsconfig.json b/ghost/stats-service/tsconfig.json new file mode 100644 index 0000000000..76f841d0e2 --- /dev/null +++ b/ghost/stats-service/tsconfig.json @@ -0,0 +1,101 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig.json to read more about this file */ + + /* Projects */ + // "incremental": true, /* Enable incremental compilation */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./", /* Specify the folder for .tsbuildinfo incremental compilation files. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h' */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using `jsx: react-jsx*`.` */ + // "reactNamespace": "", /* Specify the object invoked for `createElement`. This only applies when targeting `react` JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like `./node_modules/@types`. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "resolveJsonModule": true, /* Enable importing .json files */ + // "noResolve": true, /* Disallow `import`s, `require`s or ``s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the `checkJS` option to get errors from these files. */ + "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from `node_modules`. Only applicable with `allowJs`. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If `declaration` is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have `@internal` in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like `__extends` in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing `const enum` declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables `allowSyntheticDefaultImports` for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied `any` type.. */ + // "strictNullChecks": true, /* When type checking, take into account `null` and `undefined`. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for `bind`, `call`, and `apply` methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when `this` is given the type `any`. */ + // "useUnknownInCatchVariables": true, /* Type catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when a local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Include 'undefined' in index signature results */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +}