From e41984d0a53b5f1563e23469008251e20a7bf7c2 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Thu, 29 Aug 2024 15:39:44 +0100 Subject: [PATCH 01/10] Initial tinybird setup ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd - Tinybird has a system for managing it's configuration as code, with full ci/cd support - The tinybird CLI tool uses python, so we'll run that using docker, via `yarn tb` - Some of the files tinybird adds should not be in source control, so we've added those to git ignore - Everything in /ghost/tinybird is tinybird's init config --- .gitignore | 5 ++ ghost/tinybird/.tinyenv | 31 ++++++++++++ ghost/tinybird/README.md | 18 +++++++ ghost/tinybird/requirements.txt | 1 + ghost/tinybird/scripts/append_fixtures.sh | 21 ++++++++ ghost/tinybird/scripts/exec_test.sh | 58 +++++++++++++++++++++++ package.json | 3 +- 7 files changed, 136 insertions(+), 1 deletion(-) create mode 100644 ghost/tinybird/.tinyenv create mode 100644 ghost/tinybird/README.md create mode 100644 ghost/tinybird/requirements.txt create mode 100755 ghost/tinybird/scripts/append_fixtures.sh create mode 100755 ghost/tinybird/scripts/exec_test.sh diff --git a/.gitignore b/.gitignore index 7090507371..b3e179754d 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,8 @@ tsconfig.tsbuildinfo /apps/admin-x-settings/test-results/ /apps/admin-x-settings/playwright-report/ /apps/admin-x-settings/playwright/.cache/ + +# Tinybird +.tinyb +.venv +.diff_tmp diff --git a/ghost/tinybird/.tinyenv b/ghost/tinybird/.tinyenv new file mode 100644 index 0000000000..3281ddfae8 --- /dev/null +++ b/ghost/tinybird/.tinyenv @@ -0,0 +1,31 @@ + +# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers +# bump post to deploy to the current live Release, rollback to previous post version is not available +# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status +# bump major to deploy a new Release in preview status +VERSION=0.0.0 + + + +########## +# OPTIONAL env vars +# Deploy a new Release in preview status (default is 1) +# TB_AUTO_PROMOTE=0 + +# Check if deploy requires backfilling on preview (default is 1) +# TB_CHECK_BACKFILL_REQUIRED=0 + +# Force old Releases deletion on promote (default is 0) +# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use +# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0 + +# Don't print CLI version warning message if there's a new available version +# TB_VERSION_WARNING=0 + +# Skip regression tests +# TB_SKIP_REGRESSION=0 + +# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output. +# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>" +# OBFUSCATE_PATTERN_SEPARATOR=|| +########## diff --git a/ghost/tinybird/README.md b/ghost/tinybird/README.md new file mode 100644 index 0000000000..d3230a50d6 --- /dev/null +++ b/ghost/tinybird/README.md @@ -0,0 +1,18 @@ +# Tinybird + +This folder contains configuration for Tinybird, so that the stats feature can be used. + +We sync this configuration with Tinybird via the Tinybird CLI. + +## Tinybird CLI + +The Tinybird CLI is used via Docker. + +```bash +yarn tb +``` + +Documentation for the Tinybird CLI: https://docs.tinybird.co/v/0.22.0/cli/overview +Note: you can use python if you prefer, but we use Docker for consistency. + +How to work with version control: https://docs.tinybird.co/v/0.22.0/guides/version-control diff --git a/ghost/tinybird/requirements.txt b/ghost/tinybird/requirements.txt new file mode 100644 index 0000000000..86035df960 --- /dev/null +++ b/ghost/tinybird/requirements.txt @@ -0,0 +1 @@ +tinybird-cli>=4,<5 \ No newline at end of file diff --git a/ghost/tinybird/scripts/append_fixtures.sh b/ghost/tinybird/scripts/append_fixtures.sh new file mode 100755 index 0000000000..e8745565af --- /dev/null +++ b/ghost/tinybird/scripts/append_fixtures.sh @@ -0,0 +1,21 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +directory="datasources/fixtures" +extensions=("csv" "ndjson") + +absolute_directory=$(realpath "$directory") + +for extension in "${extensions[@]}"; do + file_list=$(find "$absolute_directory" -type f -name "*.$extension") + + for file_path in $file_list; do + file_name=$(basename "$file_path") + file_name_without_extension="${file_name%.*}" + + command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name" + echo $command + $command + done +done diff --git a/ghost/tinybird/scripts/exec_test.sh b/ghost/tinybird/scripts/exec_test.sh new file mode 100755 index 0000000000..50571d952d --- /dev/null +++ b/ghost/tinybird/scripts/exec_test.sh @@ -0,0 +1,58 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +export TB_VERSION_WARNING=0 + +run_test() { + t=$1 + echo "** Running $t **" + echo "** $(cat $t)" + tmpfile=$(mktemp) + retries=0 + TOTAL_RETRIES=3 + + # When appending fixtures, we need to retry in case of the data is not replicated in time + while [ $retries -lt $TOTAL_RETRIES ]; do + # Run the test and store the output in a temporary file + bash $t $2 >$tmpfile + exit_code=$? + if [ "$exit_code" -eq 0 ]; then + # If the test passed, break the loop + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + break + # If the test failed, increment the retries counter and try again + else + retries=$((retries+1)) + fi + # If the bash command failed, print an error message and break the loop + else + break + fi + done + + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + echo "✅ Test $t passed" + rm $tmpfile + return 0 + elif [ $retries -eq $TOTAL_RETRIES ]; then + echo "🚨 ERROR: Test $t failed, diff:"; + diff -B ${t}.result $tmpfile + rm $tmpfile + return 1 + else + echo "🚨 ERROR: Test $t failed with bash command exit code $?" + cat $tmpfile + rm $tmpfile + return 1 + fi + echo "" +} +export -f run_test + +fail=0 +find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1 + +if [ $fail == 1 ]; then + exit -1; +fi diff --git a/package.json b/package.json index b50208c85b..8a8034adfe 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "main": "yarn main:monorepo && yarn main:submodules", "main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn", "main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn\"", - "prepare": "husky install .github/hooks" + "prepare": "husky install .github/hooks", + "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker" }, "resolutions": { "@tryghost/errors": "1.3.5", From 0647172ca8398e4e350e9b7019905cfa71d38ffb Mon Sep 17 00:00:00 2001 From: Kevin Ansfield Date: Thu, 29 Aug 2024 10:47:48 +0100 Subject: [PATCH 02/10] Added style inlining to staff service emails no issue - uses same `juice` version that we use in our newsletter emails --- ghost/staff-service/lib/StaffServiceEmails.js | 6 +++++- ghost/staff-service/package.json | 7 ++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/ghost/staff-service/lib/StaffServiceEmails.js b/ghost/staff-service/lib/StaffServiceEmails.js index a4d5206176..7a970a0dfe 100644 --- a/ghost/staff-service/lib/StaffServiceEmails.js +++ b/ghost/staff-service/lib/StaffServiceEmails.js @@ -495,10 +495,14 @@ class StaffServiceEmails { sharedData = await this.getSharedData(data.recipient); } - return htmlTemplate({ + const html = htmlTemplate({ ...data, ...sharedData }); + + const juice = require('juice'); + + return juice(html, {inlinePseudoElements: true, removeStyleTags: true}); } async renderText(templateName, data) { diff --git a/ghost/staff-service/package.json b/ghost/staff-service/package.json index 7f18499201..49446d9e7d 100644 --- a/ghost/staff-service/package.json +++ b/ghost/staff-service/package.json @@ -23,9 +23,10 @@ "sinon": "15.2.0" }, "dependencies": { - "lodash": "4.17.21", - "moment": "2.29.1", + "@tryghost/email-addresses": "0.0.0", "handlebars": "4.7.8", - "@tryghost/email-addresses": "0.0.0" + "juice": "9.1.0", + "lodash": "4.17.21", + "moment": "2.29.1" } } From b79534387d65ea9a3cc46ed2d4f20e0d863f2c28 Mon Sep 17 00:00:00 2001 From: Sanne de Vries Date: Thu, 29 Aug 2024 12:47:40 +0200 Subject: [PATCH 03/10] Updated notification email mobile styles ref https://linear.app/tryghost/issue/PLG-200 --- .../recommendation-emails.test.js.snap | 624 +++++++++--------- .../lib/email-templates/donation.hbs | 4 +- .../lib/email-templates/new-free-signup.hbs | 6 +- .../email-templates/new-paid-cancellation.hbs | 6 +- .../lib/email-templates/new-paid-started.hbs | 10 +- .../lib/email-templates/partials/styles.hbs | 53 +- 6 files changed, 336 insertions(+), 367 deletions(-) diff --git a/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap index 4315ec4374..a40dc54480 100644 --- a/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap +++ b/ghost/core/test/e2e-server/services/__snapshots__/recommendation-emails.test.js.snap @@ -8,114 +8,102 @@ exports[`Incoming Recommendation Emails Sends a different email if we receive a 👍 New recommendation +@media only screen and (max-width: 620px) { + table.body h1 { + font-size: 22px !important; + padding-bottom: 16px !important; + } + + table.body p, +table.body ul, +table.body ol, +table.body td, +table.body span, +table.body a { + font-size: 16px !important; + } + + table.body .wrapper, +table.body .article { + padding: 10px !important; + } + + table.body .content { + padding: 0 !important; + } + + table.body .container { + padding: 0 !important; + width: 100% !important; + } + + table.body .main { + border-left-width: 0 !important; + border-radius: 0 !important; + border-right-width: 0 !important; + } + + table.body .img-responsive { + height: auto !important; + max-width: 100% !important; + width: auto !important; + } + + table.body p.large, +table.body p.large a { + font-size: 18px !important; + } + + table.body p.small, +table.body a.small { + font-size: 12px !important; + } + + .new-mention-thumbnail { + display: none !important; + } +} +@media all { + .ExternalClass { + width: 100%; + } + + .ExternalClass, +.ExternalClass p, +.ExternalClass span, +.ExternalClass font, +.ExternalClass td, +.ExternalClass div { + line-height: 100%; + } + + .text-link a { + color: inherit !important; + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + .text-link-accent a { + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + #MessageViewBody a { + color: inherit; + text-decoration: none; + font-size: inherit; + font-family: inherit; + font-weight: inherit; + line-height: inherit; + } +} + @@ -138,7 +126,7 @@ exports[`Incoming Recommendation Emails Sends a different email if we receive a
- +
@@ -179,7 +167,7 @@ exports[`Incoming Recommendation Emails Sends a different email if we receive a - View recommendations + View recommendations @@ -264,114 +252,102 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend 👍 New recommendation +@media only screen and (max-width: 620px) { + table.body h1 { + font-size: 22px !important; + padding-bottom: 16px !important; + } + + table.body p, +table.body ul, +table.body ol, +table.body td, +table.body span, +table.body a { + font-size: 16px !important; + } + + table.body .wrapper, +table.body .article { + padding: 10px !important; + } + + table.body .content { + padding: 0 !important; + } + + table.body .container { + padding: 0 !important; + width: 100% !important; + } + + table.body .main { + border-left-width: 0 !important; + border-radius: 0 !important; + border-right-width: 0 !important; + } + + table.body .img-responsive { + height: auto !important; + max-width: 100% !important; + width: auto !important; + } + + table.body p.large, +table.body p.large a { + font-size: 18px !important; + } + + table.body p.small, +table.body a.small { + font-size: 12px !important; + } + + .new-mention-thumbnail { + display: none !important; + } +} +@media all { + .ExternalClass { + width: 100%; + } + + .ExternalClass, +.ExternalClass p, +.ExternalClass span, +.ExternalClass font, +.ExternalClass td, +.ExternalClass div { + line-height: 100%; + } + + .text-link a { + color: inherit !important; + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + .text-link-accent a { + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + #MessageViewBody a { + color: inherit; + text-decoration: none; + font-size: inherit; + font-family: inherit; + font-weight: inherit; + line-height: inherit; + } +} + @@ -395,7 +371,7 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend
- +
@@ -436,7 +412,7 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend - Recommend back + Recommend back @@ -499,114 +475,102 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend 👍 New recommendation +@media only screen and (max-width: 620px) { + table.body h1 { + font-size: 22px !important; + padding-bottom: 16px !important; + } + + table.body p, +table.body ul, +table.body ol, +table.body td, +table.body span, +table.body a { + font-size: 16px !important; + } + + table.body .wrapper, +table.body .article { + padding: 10px !important; + } + + table.body .content { + padding: 0 !important; + } + + table.body .container { + padding: 0 !important; + width: 100% !important; + } + + table.body .main { + border-left-width: 0 !important; + border-radius: 0 !important; + border-right-width: 0 !important; + } + + table.body .img-responsive { + height: auto !important; + max-width: 100% !important; + width: auto !important; + } + + table.body p.large, +table.body p.large a { + font-size: 18px !important; + } + + table.body p.small, +table.body a.small { + font-size: 12px !important; + } + + .new-mention-thumbnail { + display: none !important; + } +} +@media all { + .ExternalClass { + width: 100%; + } + + .ExternalClass, +.ExternalClass p, +.ExternalClass span, +.ExternalClass font, +.ExternalClass td, +.ExternalClass div { + line-height: 100%; + } + + .text-link a { + color: inherit !important; + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + .text-link-accent a { + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } + + #MessageViewBody a { + color: inherit; + text-decoration: none; + font-size: inherit; + font-family: inherit; + font-weight: inherit; + line-height: inherit; + } +} + @@ -630,7 +594,7 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend
- +
@@ -671,7 +635,7 @@ exports[`Incoming Recommendation Emails Sends an email if we receive a recommend - Recommend back + Recommend back diff --git a/ghost/staff-service/lib/email-templates/donation.hbs b/ghost/staff-service/lib/email-templates/donation.hbs index 76c10e231b..c1896b924e 100644 --- a/ghost/staff-service/lib/email-templates/donation.hbs +++ b/ghost/staff-service/lib/email-templates/donation.hbs @@ -37,9 +37,9 @@

From:

-

{{donation.name}} {{#if memberData}}• View{{/if}}

+

{{donation.name}} {{#if memberData}}• View{{/if}}

Amount received:

-

{{donation.amount}}

+

{{donation.amount}}

{{#if donation.donationMessage}}

“{{donation.donationMessage}}”

{{/if}} diff --git a/ghost/staff-service/lib/email-templates/new-free-signup.hbs b/ghost/staff-service/lib/email-templates/new-free-signup.hbs index 3263e6a750..db82aa1f51 100644 --- a/ghost/staff-service/lib/email-templates/new-free-signup.hbs +++ b/ghost/staff-service/lib/email-templates/new-free-signup.hbs @@ -42,13 +42,13 @@

Name:

-

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

+

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

{{#if referrerSource}}

Source:

-

{{referrerSource}}

+

{{referrerSource}}

{{#if attributionTitle}}

Page:

-

{{attributionTitle}}

+

{{attributionTitle}}

{{/if}} {{/if}} diff --git a/ghost/staff-service/lib/email-templates/new-paid-cancellation.hbs b/ghost/staff-service/lib/email-templates/new-paid-cancellation.hbs index d5050d0aab..5d0aabb3d3 100644 --- a/ghost/staff-service/lib/email-templates/new-paid-cancellation.hbs +++ b/ghost/staff-service/lib/email-templates/new-paid-cancellation.hbs @@ -46,15 +46,15 @@

Name:

-

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

+

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

Tier:

-

{{tierData.name}}{{#if tierData.details}} • {{tierData.details}}{{/if}}

+

{{tierData.name}}{{#if tierData.details}} • {{tierData.details}}{{/if}}

{{#if subscriptionData.cancelNow}}

Expired on:

{{else}}

Expires on:

{{/if}} -

{{subscriptionData.expiryAt}}

+

{{subscriptionData.expiryAt}}

{{#if subscriptionData.cancellationReason}}

"{{subscriptionData.cancellationReason}}"

{{/if}} diff --git a/ghost/staff-service/lib/email-templates/new-paid-started.hbs b/ghost/staff-service/lib/email-templates/new-paid-started.hbs index 8d6d96db63..38cc764c65 100644 --- a/ghost/staff-service/lib/email-templates/new-paid-started.hbs +++ b/ghost/staff-service/lib/email-templates/new-paid-started.hbs @@ -42,19 +42,19 @@

Name:

-

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

+

{{memberData.name}}{{#if memberData.showEmail}} ({{memberData.email}}){{/if}}

Tier:

-

{{tierData.name}}{{#if tierData.details}} • {{tierData.details}}{{/if}}

+

{{tierData.name}}{{#if tierData.details}} • {{tierData.details}}{{/if}}

{{#if offerData}}

Offer:

-

{{offerData.name}} • {{offerData.details}}

+

{{offerData.name}} • {{offerData.details}}

{{/if}} {{#if referrerSource}}

Source:

-

{{referrerSource}}

+

{{referrerSource}}

{{#if attributionTitle}}

Page:

-

{{attributionTitle}}

+

{{attributionTitle}}

{{/if}} {{/if}} diff --git a/ghost/staff-service/lib/email-templates/partials/styles.hbs b/ghost/staff-service/lib/email-templates/partials/styles.hbs index 10a74ed21c..7de00b563c 100644 --- a/ghost/staff-service/lib/email-templates/partials/styles.hbs +++ b/ghost/staff-service/lib/email-templates/partials/styles.hbs @@ -3,48 +3,46 @@ RESPONSIVE AND MOBILE FRIENDLY STYLES ------------------------------------- */ @media only screen and (max-width: 620px) { - table[class=body] h1 { - font-size: 28px !important; - margin-bottom: 10px !important; + table.body h1 { + font-size: 22px !important; + padding-bottom: 16px !important; } - table[class=body] p, - table[class=body] ul, - table[class=body] ol, - table[class=body] td, - table[class=body] span, - table[class=body] a { + table.body p, + table.body ul, + table.body ol, + table.body td, + table.body span, + table.body a { font-size: 16px !important; } - table[class=body] .wrapper, - table[class=body] .article { + table.body .wrapper, + table.body .article { padding: 10px !important; } - table[class=body] .content { + table.body .content { padding: 0 !important; } - table[class=body] .container { + table.body .container { padding: 0 !important; width: 100% !important; } - table[class=body] .main { + table.body .main { border-left-width: 0 !important; border-radius: 0 !important; border-right-width: 0 !important; } - table[class=body] .btn table { - width: 100% !important; - } - table[class=body] .btn a { - width: 100% !important; - } - table[class=body] .img-responsive { + table.body .img-responsive { height: auto !important; max-width: 100% !important; width: auto !important; } - table[class=body] p[class=small], - table[class=body] a[class=small] { - font-size: 11px !important; + table.body p.large, + table.body p.large a { + font-size: 18px !important; + } + table.body p.small, + table.body a.small { + font-size: 12px !important; } .new-mention-thumbnail { display: none !important; @@ -74,6 +72,13 @@ line-height: inherit !important; text-decoration: none !important; } + .text-link-accent a { + font-family: inherit !important; + font-size: inherit !important; + font-weight: inherit !important; + line-height: inherit !important; + text-decoration: none !important; + } #MessageViewBody a { color: inherit; text-decoration: none; From 79f4b523ac33d14ca1acb42d52729dbab712568f Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Thu, 29 Aug 2024 17:13:05 +0100 Subject: [PATCH 04/10] Added analytics_events tinybird datasource ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd ref https://www.tinybird.co/docs/concepts/data-sources - This is our main datasource, where we'll store events that come in as people browse around Ghost - It's defined using tinybird's format, and then deployed out to tinybird using `tb deploy` --- .../datasources/analytics_events.datasource | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ghost/tinybird/datasources/analytics_events.datasource diff --git a/ghost/tinybird/datasources/analytics_events.datasource b/ghost/tinybird/datasources/analytics_events.datasource new file mode 100644 index 0000000000..a812f3a48f --- /dev/null +++ b/ghost/tinybird/datasources/analytics_events.datasource @@ -0,0 +1,14 @@ +DESCRIPTION > + Analytics events landing data source + +SCHEMA > + `timestamp` DateTime `json:$.timestamp`, + `session_id` String `json:$.session_id`, + `action` LowCardinality(String) `json:$.action`, + `version` LowCardinality(String) `json:$.version`, + `payload` String `json:$.payload` + +ENGINE MergeTree +ENGINE_PARTITION_KEY toYYYYMM(timestamp) +ENGINE_SORTING_KEY timestamp +ENGINE_TTL timestamp + toIntervalDay(60) From c2ae91e4db184d64c17bcd63f6e4e32343910fd1 Mon Sep 17 00:00:00 2001 From: Steve Larson <9larsons@gmail.com> Date: Thu, 29 Aug 2024 12:51:23 -0500 Subject: [PATCH 05/10] Added config flag to disable recommendations service (#20879) no ref This service can get rather noisy when doing local development with our data generator, as we do not use real urls, and therefore generate a lot of not found errors in the console. --- .../recommendations/RecommendationServiceWrapper.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js index 56372c495a..25c10a9343 100644 --- a/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js +++ b/ghost/core/core/server/services/recommendations/RecommendationServiceWrapper.js @@ -39,11 +39,16 @@ class RecommendationServiceWrapper { incomingRecommendationService; init() { + const config = require('../../../shared/config'); + if (config.get('services:recommendations:enabled') === false) { + logging.info('[Recommendations] Service is disabled via config'); + return; + } + if (this.repository) { return; } - const config = require('../../../shared/config'); const urlUtils = require('../../../shared/url-utils'); const models = require('../../models'); const sentry = require('../../../shared/sentry'); From d30164df979e165dab843b0b9087088533cb16eb Mon Sep 17 00:00:00 2001 From: Sodbileg Gansukh Date: Fri, 30 Aug 2024 03:17:16 +0800 Subject: [PATCH 06/10] =?UTF-8?q?=E2=9C=A8=20Improved=20publishing=20flow?= =?UTF-8?q?=20(#20878)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ref DES-706 * After a user publishes or schedules a post, they are directed to the post list * If a post is sent as an email, they are directed to the Analytics page * In both cases, a confirmation modal is shown * If a post is published, they can share it directly from the confirmation modal * Added a "Share" button and some additional functions (view, edit, and delete post) to published posts in post analytics * Added a manual "Refresh" button to post analytics so that there is no need to reload the whole app to update the data --------- Co-authored-by: Sag --- .../settings/advanced/labs/AlphaFeatures.tsx | 8 -- .../components/editor/modals/publish-flow.hbs | 10 +- .../editor/modals/publish-flow/confirm.js | 43 ++++--- .../app/components/modal-post-success.hbs | 6 +- .../app/components/posts-list/list-item.hbs | 4 +- ghost/admin/app/components/posts-list/list.js | 4 +- .../admin/app/components/posts/analytics.hbs | 110 ++++++++---------- ghost/admin/app/components/posts/analytics.js | 10 +- ghost/admin/app/services/feature.js | 2 - ghost/core/core/shared/labs.js | 4 +- .../admin/__snapshots__/config.test.js.snap | 2 - .../test/e2e-browser/admin/publishing.spec.js | 18 ++- 12 files changed, 90 insertions(+), 131 deletions(-) diff --git a/apps/admin-x-settings/src/components/settings/advanced/labs/AlphaFeatures.tsx b/apps/admin-x-settings/src/components/settings/advanced/labs/AlphaFeatures.tsx index f71f559524..fd20996e7c 100644 --- a/apps/admin-x-settings/src/components/settings/advanced/labs/AlphaFeatures.tsx +++ b/apps/admin-x-settings/src/components/settings/advanced/labs/AlphaFeatures.tsx @@ -59,14 +59,6 @@ const features = [{ title: 'Content Visibility', description: 'Enables content visibility in Emails', flag: 'contentVisibility' -},{ - title: 'Publish Flow — End Screen', - description: 'Enables improved publish flow', - flag: 'publishFlowEndScreen' -},{ - title: 'Post Analytics — Refresh', - description: 'Adds a refresh button to the post analytics screen', - flag: 'postAnalyticsRefresh' }]; const AlphaFeatures: React.FC = () => { diff --git a/ghost/admin/app/components/editor/modals/publish-flow.hbs b/ghost/admin/app/components/editor/modals/publish-flow.hbs index e202eb7bff..c964c637b3 100644 --- a/ghost/admin/app/components/editor/modals/publish-flow.hbs +++ b/ghost/admin/app/components/editor/modals/publish-flow.hbs @@ -1,6 +1,6 @@
- @@ -45,14 +45,6 @@ @close={{@close}} /> {{else if this.isComplete}} - {{#unless (feature "publishFlowEndScreen")}} - - {{/unless}} {{else}}
{{/if}} - +

{{this.post.title}}

{{#if this.post.excerpt}} @@ -103,10 +103,10 @@ {{/if}} - + {{#unless this.post.emailOnly}} - +
{{#if this.post.featureImage}}
diff --git a/ghost/admin/app/components/posts-list/list-item.hbs b/ghost/admin/app/components/posts-list/list-item.hbs index 6b201b9b30..fa1e1f5424 100644 --- a/ghost/admin/app/components/posts-list/list-item.hbs +++ b/ghost/admin/app/components/posts-list/list-item.hbs @@ -32,7 +32,7 @@ {{/if}}

-

+

Published {{#if @post.hasEmail}} @@ -87,7 +87,7 @@ – Lexical {{/if}} --}}

-

+

{{#if @post.isScheduled}} Scheduled diff --git a/ghost/admin/app/components/posts-list/list.js b/ghost/admin/app/components/posts-list/list.js index ce99157312..84a730de21 100644 --- a/ghost/admin/app/components/posts-list/list.js +++ b/ghost/admin/app/components/posts-list/list.js @@ -16,9 +16,7 @@ export default class PostsList extends Component { constructor() { super(...arguments); - if (this.feature.publishFlowEndScreen) { - this.checkPublishFlowModal(); - } + this.checkPublishFlowModal(); } async checkPublishFlowModal() { diff --git a/ghost/admin/app/components/posts/analytics.hbs b/ghost/admin/app/components/posts/analytics.hbs index 9e57b4e9b7..3901c6978d 100644 --- a/ghost/admin/app/components/posts/analytics.hbs +++ b/ghost/admin/app/components/posts/analytics.hbs @@ -34,66 +34,58 @@ {{moment-format publishedAt "HH:mm"}} {{/let}}

- {{#if (feature "publishFlowEndScreen")}} -
diff --git a/ghost/admin/app/components/posts/analytics.js b/ghost/admin/app/components/posts/analytics.js index b4f0daf64b..d95f3359a6 100644 --- a/ghost/admin/app/components/posts/analytics.js +++ b/ghost/admin/app/components/posts/analytics.js @@ -50,9 +50,7 @@ export default class Analytics extends Component { constructor() { super(...arguments); - if (this.feature.publishFlowEndScreen) { - this.checkPublishFlowModal(); - } + this.checkPublishFlowModal(); } openPublishFlowModal() { @@ -73,11 +71,7 @@ export default class Analytics extends Component { } get post() { - if (this.feature.publishFlowEndScreen) { - return this._post ?? this.args.post; - } - - return this.args.post; + return this._post ?? this.args.post; } set post(value) { diff --git a/ghost/admin/app/services/feature.js b/ghost/admin/app/services/feature.js index 14be59d768..09f4042a45 100644 --- a/ghost/admin/app/services/feature.js +++ b/ghost/admin/app/services/feature.js @@ -78,8 +78,6 @@ export default class FeatureService extends Service { @feature('ActivityPub') ActivityPub; @feature('editorExcerpt') editorExcerpt; @feature('contentVisibility') contentVisibility; - @feature('publishFlowEndScreen') publishFlowEndScreen; - @feature('postAnalyticsRefresh') postAnalyticsRefresh; _user = null; diff --git a/ghost/core/core/shared/labs.js b/ghost/core/core/shared/labs.js index 4f242c4ccc..e9a1a7ad68 100644 --- a/ghost/core/core/shared/labs.js +++ b/ghost/core/core/shared/labs.js @@ -45,9 +45,7 @@ const ALPHA_FEATURES = [ 'importMemberTier', 'lexicalIndicators', 'adminXDemo', - 'contentVisibility', - 'publishFlowEndScreen', - 'postAnalyticsRefresh' + 'contentVisibility' ]; module.exports.GA_KEYS = [...GA_FEATURES]; diff --git a/ghost/core/test/e2e-api/admin/__snapshots__/config.test.js.snap b/ghost/core/test/e2e-api/admin/__snapshots__/config.test.js.snap index 44ddb6609b..f45ded2b64 100644 --- a/ghost/core/test/e2e-api/admin/__snapshots__/config.test.js.snap +++ b/ghost/core/test/e2e-api/admin/__snapshots__/config.test.js.snap @@ -29,8 +29,6 @@ Object { "members": true, "newEmailAddresses": true, "outboundLinkTagging": true, - "postAnalyticsRefresh": true, - "publishFlowEndScreen": true, "stripeAutomaticTax": true, "themeErrorsNotification": true, "tipsAndDonations": true, diff --git a/ghost/core/test/e2e-browser/admin/publishing.spec.js b/ghost/core/test/e2e-browser/admin/publishing.spec.js index 788a08c2c2..0b20570143 100644 --- a/ghost/core/test/e2e-browser/admin/publishing.spec.js +++ b/ghost/core/test/e2e-browser/admin/publishing.spec.js @@ -11,11 +11,11 @@ const {createTier, createMember, createPostDraft, impersonateMember} = require(' * @param {string} [hoverStatus] Optional different status when you hover the status */ const checkPostStatus = async (page, status, hoverStatus) => { - await expect(page.locator('[data-test-editor-post-status]')).toContainText(status, {timeout: 5000}); + await expect(page.locator('[data-test-editor-post-status]').first()).toContainText(status, {timeout: 5000}); if (hoverStatus) { - await page.locator('[data-test-editor-post-status]').hover(); - await expect(page.locator('[data-test-editor-post-status]')).toContainText(hoverStatus, {timeout: 5000}); + await page.locator('[data-test-editor-post-status]').first().hover(); + await expect(page.locator('[data-test-editor-post-status]').first()).toContainText(hoverStatus, {timeout: 5000}); } }; @@ -198,8 +198,6 @@ test.describe('Publishing', () => { await createPostDraft(sharedPage, postData); await publishPost(sharedPage, {type: 'publish+send'}); await closePublishFlow(sharedPage); - - await checkPostStatus(sharedPage, 'Published'); await checkPostPublished(sharedPage, postData); }); @@ -232,8 +230,6 @@ test.describe('Publishing', () => { await createPostDraft(sharedPage, postData); await publishPost(sharedPage, {type: 'send'}); await closePublishFlow(sharedPage); - await checkPostStatus(sharedPage, 'Sent to '); // can't test for 1 member for now, because depends on test ordering :( (sometimes 2 members are created) - await checkPostNotPublished(sharedPage, postData); }); }); @@ -327,8 +323,9 @@ test.describe('Publishing', () => { await expect(publishedHeader).toContainText(date.toFormat('LLL d, yyyy')); // add some extra text to the post + await adminPage.locator('li[data-test-post-id]').first().click(); await adminPage.locator('[data-kg="editor"]').first().click(); - await adminPage.waitForTimeout(200); // + await adminPage.waitForTimeout(500); await adminPage.keyboard.type(' This is some updated text.'); // change some post settings @@ -431,7 +428,7 @@ test.describe('Publishing', () => { // Schedule the post to publish asap (by setting it to 00:00, it will get auto corrected to the minimum time possible - 5 seconds in the future) await publishPost(sharedPage, {type: 'send', time: '00:00'}); await closePublishFlow(sharedPage); - await checkPostStatus(sharedPage, 'Scheduled', 'Scheduled to be sent to'); + await checkPostStatus(sharedPage, 'Scheduled', 'Scheduled to be sent in a few seconds'); const editorUrl = await sharedPage.url(); // Check not published yet @@ -472,6 +469,7 @@ test.describe('Publishing', () => { await checkPostNotPublished(testsharedPage, postData); // Now unschedule this post + await sharedPage.locator('li[data-test-post-id]').first().click(); await sharedPage.locator('[data-test-button="update-flow"]').first().click(); await sharedPage.locator('[data-test-button="revert-to-draft"]').click(); @@ -566,6 +564,7 @@ test.describe('Updating post access', () => { // publish await publishPost(sharedPage); + await closePublishFlow(sharedPage); const frontendPage = await openPublishedPostBookmark(sharedPage); // non-member doesn't have access @@ -607,7 +606,6 @@ test.describe('Updating post access', () => { await closePublishFlow(page); // go to settings and change the timezone - await page.locator('[data-test-link="posts"]').click(); await page.locator('[data-test-nav="settings"]').click(); await expect(page.getByTestId('timezone')).toContainText('UTC'); From f79f5471b4d3f635f1800b790128335dcb6e9282 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Thu, 29 Aug 2024 21:40:41 +0100 Subject: [PATCH 07/10] Added stats tracker script to ghost head (#20881) closes https://linear.app/tryghost/issue/ANAL-9/initial-tracker-in-ghost-head - Given that all of the correct config is in place, output a tracking script - This allows us to send pageview events into tinybird - All of the details (location of the script, destination etc) are kept in config so that it's easy to change for different environments --- .../core/core/frontend/helpers/ghost_head.js | 19 ++ .../__snapshots__/ghost_head.test.js.snap | 256 +++++++++++++++++- .../unit/frontend/helpers/ghost_head.test.js | 89 ++++++ 3 files changed, 357 insertions(+), 7 deletions(-) diff --git a/ghost/core/core/frontend/helpers/ghost_head.js b/ghost/core/core/frontend/helpers/ghost_head.js index 09904905ce..9661145e09 100644 --- a/ghost/core/core/frontend/helpers/ghost_head.js +++ b/ghost/core/core/frontend/helpers/ghost_head.js @@ -141,6 +141,21 @@ function getWebmentionDiscoveryLink() { } } +function getTinybirdTrackerScript(dataRoot) { + const scriptUrl = config.get('tinybird:tracker:scriptUrl'); + const endpoint = config.get('tinybird:tracker:endpoint'); + const token = config.get('tinybird:tracker:token'); + + const tbParams = _.map({ + site_uuid: config.get('tinybird:tracker:id'), + post_uuid: dataRoot.post?.uuid, + member_uuid: dataRoot.member?.uuid, + member_status: dataRoot.member?.status + }, (value, key) => `tb_${key}="${value}"`).join(' '); + + return ``; +} + /** * **NOTE** * Express adds `_locals`, see https://github.com/expressjs/express/blob/4.15.4/lib/response.js#L962. @@ -319,6 +334,10 @@ module.exports = async function ghost_head(options) { // eslint-disable-line cam if (!_.isEmpty(tagCodeInjection)) { head.push(tagCodeInjection); } + + if (config.get('tinybird') && config.get('tinybird:tracker') && config.get('tinybird:tracker:scriptUrl')) { + head.push(getTinybirdTrackerScript(dataRoot)); + } } debug('end'); diff --git a/ghost/core/test/unit/frontend/helpers/__snapshots__/ghost_head.test.js.snap b/ghost/core/test/unit/frontend/helpers/__snapshots__/ghost_head.test.js.snap index d38ac6af6e..b9bd03697e 100644 --- a/ghost/core/test/unit/frontend/helpers/__snapshots__/ghost_head.test.js.snap +++ b/ghost/core/test/unit/frontend/helpers/__snapshots__/ghost_head.test.js.snap @@ -457,7 +457,7 @@ Object { "string": " - + @@ -469,7 +469,7 @@ Object { - + ", } `; @@ -959,6 +959,248 @@ Object { } `; +exports[`{{ghost_head}} helper includes tinybird tracker script when config is set Sets tb_post_uuid on post page 1 1`] = ` +Object { + "rendered": " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ", +} +`; + +exports[`{{ghost_head}} helper includes tinybird tracker script when config is set sets both tb_member_x variables and tb_post_uuid on logged in post page 1 1`] = ` +Object { + "rendered": " + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ", +} +`; + +exports[`{{ghost_head}} helper includes tinybird tracker script when config is set sets tb_member_x variables on logged in home page 1 1`] = ` +Object { + "rendered": " + + + + + + + + + + + + + + + + + + + + + + + + ", +} +`; + +exports[`{{ghost_head}} helper includes tinybird tracker script when config is set with all tb_variables set to undefined on logged out home page 1 1`] = ` +Object { + "rendered": " + + + + + + + + + + + + + + + + + + + + + + + + ", +} +`; + exports[`{{ghost_head}} helper members scripts includes portal when members enabled 1 1`] = ` Object { "rendered": " diff --git a/ghost/core/test/unit/frontend/helpers/ghost_head.test.js b/ghost/core/test/unit/frontend/helpers/ghost_head.test.js index 2d07430fd7..2241abb0a9 100644 --- a/ghost/core/test/unit/frontend/helpers/ghost_head.test.js +++ b/ghost/core/test/unit/frontend/helpers/ghost_head.test.js @@ -340,6 +340,19 @@ describe('{{ghost_head}} helper', function () { published_at: new Date(0), updated_at: new Date(0) })); + + posts.push(createPost({ // Post 10 + title: 'Testing stats', + uuid: 'post_uuid', + excerpt: 'Creating stats for the site', + mobiledoc: testUtils.DataGenerator.markdownToMobiledoc('Creating stats for the site'), + authors: [ + authors[3] + ], + primary_author: authors[3], + published_at: new Date(0), + updated_at: new Date(0) + })); }; before(function () { @@ -1185,4 +1198,80 @@ describe('{{ghost_head}} helper', function () { })); }); }); + + describe('includes tinybird tracker script when config is set', function () { + beforeEach(function () { + configUtils.set({ + tinybird: { + tracker: { + scriptUrl: 'https://unpkg.com/@tinybirdco/flock.js', + endpoint: 'https://api.tinybird.co', + token: 'tinybird_token', + id: 'tb_test_site_uuid' + } + } + }); + }); + it('with all tb_variables set to undefined on logged out home page', async function () { + await testGhostHead(testUtils.createHbsResponse({ + locals: { + relativeUrl: '/', + context: ['home', 'index'], + safeVersion: '4.3' + } + })); + }); + + it('Sets tb_post_uuid on post page', async function () { + const renderObject = { + post: posts[10] + }; + + await testGhostHead(testUtils.createHbsResponse({ + renderObject: renderObject, + locals: { + relativeUrl: '/post/', + context: ['post'], + safeVersion: '0.3' + } + })); + }); + + it('sets tb_member_x variables on logged in home page', async function () { + const renderObject = { + member: { + uuid: 'member_uuid', + status: 'paid' + } + }; + + await testGhostHead(testUtils.createHbsResponse({ + renderObject: renderObject, + locals: { + relativeUrl: '/', + context: ['home', 'index'], + safeVersion: '4.3' + } + })); + }); + + it('sets both tb_member_x variables and tb_post_uuid on logged in post page', async function () { + const renderObject = { + member: { + uuid: 'member_uuid', + status: 'free' + }, + post: posts[10] + }; + + await testGhostHead(testUtils.createHbsResponse({ + renderObject: renderObject, + locals: { + relativeUrl: '/post/', + context: ['post'], + safeVersion: '4.3' + } + })); + }); + }); }); From 08bf49eaecee7af6004da240f4c5e40037fcb77c Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Thu, 29 Aug 2024 22:03:31 +0100 Subject: [PATCH 08/10] Added full suite of tinybird datasources and pipes (#20882) ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd ref https://github.com/tinybirdco/web-analytics-starter-kit/blob/main/tinybird/pipes/analytics_sessions.pipe - These datasources and pipes work together to define the main endpoints we need for our stats dashboard - They are based on the web analytics starter kit from tinybird - We've updated them to handle site_uuid - There's more to do to pipe the member-related and post-related data through the system yet --- .../datasources/analytics_pages_mv.datasource | 18 +++ .../analytics_sessions_mv.datasource | 14 ++ .../analytics_sources_mv.datasource | 13 ++ ghost/tinybird/datasources/fixtures/README.md | 15 ++ .../fixtures/mockingbird-schema.json | 72 ++++++++++ ghost/tinybird/pipes/analytics_hits.pipe | 71 ++++++++++ ghost/tinybird/pipes/analytics_pages.pipe | 24 ++++ ghost/tinybird/pipes/analytics_sessions.pipe | 20 +++ ghost/tinybird/pipes/analytics_sources.pipe | 21 +++ ghost/tinybird/pipes/kpis.pipe | 130 ++++++++++++++++++ ghost/tinybird/pipes/top_browsers.pipe | 32 +++++ ghost/tinybird/pipes/top_devices.pipe | 33 +++++ ghost/tinybird/pipes/top_locations.pipe | 32 +++++ ghost/tinybird/pipes/top_pages.pipe | 38 +++++ ghost/tinybird/pipes/top_sources.pipe | 33 +++++ ghost/tinybird/pipes/trend.pipe | 36 +++++ 16 files changed, 602 insertions(+) create mode 100644 ghost/tinybird/datasources/analytics_pages_mv.datasource create mode 100644 ghost/tinybird/datasources/analytics_sessions_mv.datasource create mode 100644 ghost/tinybird/datasources/analytics_sources_mv.datasource create mode 100644 ghost/tinybird/datasources/fixtures/README.md create mode 100644 ghost/tinybird/datasources/fixtures/mockingbird-schema.json create mode 100644 ghost/tinybird/pipes/analytics_hits.pipe create mode 100644 ghost/tinybird/pipes/analytics_pages.pipe create mode 100644 ghost/tinybird/pipes/analytics_sessions.pipe create mode 100644 ghost/tinybird/pipes/analytics_sources.pipe create mode 100644 ghost/tinybird/pipes/kpis.pipe create mode 100644 ghost/tinybird/pipes/top_browsers.pipe create mode 100644 ghost/tinybird/pipes/top_devices.pipe create mode 100644 ghost/tinybird/pipes/top_locations.pipe create mode 100644 ghost/tinybird/pipes/top_pages.pipe create mode 100644 ghost/tinybird/pipes/top_sources.pipe create mode 100644 ghost/tinybird/pipes/trend.pipe diff --git a/ghost/tinybird/datasources/analytics_pages_mv.datasource b/ghost/tinybird/datasources/analytics_pages_mv.datasource new file mode 100644 index 0000000000..c481ec4ef4 --- /dev/null +++ b/ghost/tinybird/datasources/analytics_pages_mv.datasource @@ -0,0 +1,18 @@ +SCHEMA > + `site_uuid` String, + `member_uuid` String, + `member_status` String, + `post_uuid` String, + `date` Date, + `device` String, + `browser` String, + `location` String, + `pathname` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count), + `logged_in_hits` AggregateFunction(count), + `logged_out_hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid diff --git a/ghost/tinybird/datasources/analytics_sessions_mv.datasource b/ghost/tinybird/datasources/analytics_sessions_mv.datasource new file mode 100644 index 0000000000..f5a0fe32de --- /dev/null +++ b/ghost/tinybird/datasources/analytics_sessions_mv.datasource @@ -0,0 +1,14 @@ +SCHEMA > + `site_uuid` String, + `date` Date, + `session_id` String, + `device` SimpleAggregateFunction(any, String), + `browser` SimpleAggregateFunction(any, String), + `location` SimpleAggregateFunction(any, String), + `first_hit` SimpleAggregateFunction(min, DateTime), + `latest_hit` SimpleAggregateFunction(max, DateTime), + `hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, session_id, site_uuid diff --git a/ghost/tinybird/datasources/analytics_sources_mv.datasource b/ghost/tinybird/datasources/analytics_sources_mv.datasource new file mode 100644 index 0000000000..91f1dfe3d4 --- /dev/null +++ b/ghost/tinybird/datasources/analytics_sources_mv.datasource @@ -0,0 +1,13 @@ +SCHEMA > + `site_uuid` String, + `date` Date, + `device` String, + `browser` String, + `location` String, + `referrer` String, + `visits` AggregateFunction(uniq, String), + `hits` AggregateFunction(count) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY date, device, browser, location, referrer, site_uuid diff --git a/ghost/tinybird/datasources/fixtures/README.md b/ghost/tinybird/datasources/fixtures/README.md new file mode 100644 index 0000000000..622c39c9b5 --- /dev/null +++ b/ghost/tinybird/datasources/fixtures/README.md @@ -0,0 +1,15 @@ +# Datasource fixtures + +The file mockingbird-schema.json is a schema for generating fake data using the Mockingbird CLI. + +The CLI is installed via npm: + +``` +npm install -g @tinybirdco/mockingbird +``` + +The command I'm currently using to generate the data is: + +``` +mockingbird-cli tinybird --schema ghost/tinybird/datasources/fixtures/mockingbird-schema.json --endpoint gcp_europe_west3 --token xxxx --datasource analytics_events --eps 50 --limit 5000 +``` diff --git a/ghost/tinybird/datasources/fixtures/mockingbird-schema.json b/ghost/tinybird/datasources/fixtures/mockingbird-schema.json new file mode 100644 index 0000000000..0475ca0e0b --- /dev/null +++ b/ghost/tinybird/datasources/fixtures/mockingbird-schema.json @@ -0,0 +1,72 @@ +{ + "timestamp": { + "type": "mockingbird.datetimeBetween", + "params": [ + { + "start": "2024-07-01T00:00:00.000Z", + "end": "2024-08-20T12:00:00.000Z" + } + ] + }, + "session_id": { + "type": "string.uuid" + }, + "action": { + "type": "mockingbird.pick", + "params": [ + { + "values": [ + "page_hit" + ] + } + ] + }, + "version": { + "type": "mockingbird.pick", + "params": [ + { + "values": [ + "1" + ] + } + ] + }, + "payload": { + "type": "mockingbird.pickWeighted", + "params": [ + { + "values": [ + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://web-analytics.ghost.is/\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/hello-world/\", \"href\":\"https://web-analytics.ghost.is/hello-world/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/\", \"href\":\"https://web-analytics.ghost.is/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/about/\", \"href\":\"https://web-analytics.ghost.is/about/\"}", + "{\"site_uuid\":\"mock_site_uuid\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/coming-soon/\", \"href\":\"https://web-analytics.ghost.is/coming-soon/\"}", + + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.79 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)\", \"locale\":\"en-US\", \"referrer\":\"https://www.kike.io\", \"pathname\":\"/products/\", \"href\":\"https://fake-site.ghost.is/products/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/104.0.5112.79 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IT\", \"referrer\":\"https://www.hn.com\", \"pathname\":\"/blog/\", \"href\":\"https://fake-site.ghost.is/blog/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-GB\", \"location\":\"ES\", \"referrer\":\"\", \"pathname\":\"/contact/\", \"href\":\"https://fake-site.ghost.is/contact/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:103.0) Gecko/20100101 Firefox/103.0\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/faq/\", \"href\":\"https://fake-site.ghost.is/faq/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://fake-site.ghost.is/\", \"pathname\":\"/services/\", \"href\":\"https://fake-site.ghost.is/services/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.102 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/team/\", \"href\":\"https://fake-site.ghost.is/team/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"IL\", \"referrer\":\"https://www.google.com\", \"pathname\":\"/pricing/\", \"href\":\"https://fake-site.ghost.is/pricing/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1\", \"locale\":\"es-ES\", \"location\":\"ES\", \"referrer\":\"https://www.twitter.com\", \"pathname\":\"/resources/\", \"href\":\"https://fake-site.ghost.is/resources/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"GB\", \"referrer\":\"https://www.facebook.com\", \"pathname\":\"/careers/\", \"href\":\"https://fake-site.ghost.is/careers/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36\", \"locale\":\"en-US\", \"location\":\"CH\", \"referrer\":\"https://www.qq.ch\", \"pathname\":\"/support/\", \"href\":\"https://fake-site.ghost.is/support/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"US\", \"referrer\":\"https://www.yandex.com\", \"pathname\":\"/partners/\", \"href\":\"https://fake-site.ghost.is/partners/\"}", + "{\"site_uuid\":\"fake_site_id\", \"user-agent\":\"Mozilla/5.0 (Linux; Android 13; SM-A102U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.5249.118 Mobile Safari/537.36\", \"locale\":\"en-US\", \"location\":\"FR\", \"referrer\":\"https://www.github.com\", \"pathname\":\"/events/\", \"href\":\"https://fake-site.ghost.is/events/\"}" + ], + "weights": [ + 200, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 400, + 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100 + ] + } + ] + } + } diff --git a/ghost/tinybird/pipes/analytics_hits.pipe b/ghost/tinybird/pipes/analytics_hits.pipe new file mode 100644 index 0000000000..08fbd7023d --- /dev/null +++ b/ghost/tinybird/pipes/analytics_hits.pipe @@ -0,0 +1,71 @@ +DESCRIPTION > + Parsed `page_hit` events, implementing `browser` and `device` detection logic. + +TOKEN "dashboard" READ + +NODE parsed_hits +DESCRIPTION > + Parse raw page_hit events + +SQL > + SELECT + timestamp, + action, + version, + coalesce(session_id, '0') as session_id, + JSONExtractString(payload, 'locale') as locale, + JSONExtractString(payload, 'location') as location, + JSONExtractString(payload, 'referrer') as referrer, + JSONExtractString(payload, 'pathname') as pathname, + JSONExtractString(payload, 'href') as href, + JSONExtractString(payload, 'site_uuid') as site_uuid, + JSONExtractString(payload, 'member_uuid') as member_uuid, + JSONExtractString(payload, 'member_status') as member_status, + JSONExtractString(payload, 'post_uuid') as post_uuid, + lower(JSONExtractString(payload, 'user-agent')) as user_agent + FROM analytics_events + where action = 'page_hit' + +NODE endpoint +SQL > + SELECT + site_uuid, + timestamp, + action, + version, + session_id, + case + when member_uuid REGEXP '^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$' + then true + else false + END as logged_in, + member_uuid, + member_status, + post_uuid, + location, + referrer, + pathname, + href, + case + when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') + then 'bot' + when match(user_agent, 'android') + then 'mobile-android' + when match(user_agent, 'ipad|iphone|ipod') + then 'mobile-ios' + else 'desktop' + END as device, + case + when match(user_agent, 'firefox') + then 'firefox' + when match(user_agent, 'chrome|crios') + then 'chrome' + when match(user_agent, 'opera') + then 'opera' + when match(user_agent, 'msie|trident') + then 'ie' + when match(user_agent, 'iphone|ipad|safari') + then 'safari' + else 'Unknown' + END as browser + FROM parsed_hits diff --git a/ghost/tinybird/pipes/analytics_pages.pipe b/ghost/tinybird/pipes/analytics_pages.pipe new file mode 100644 index 0000000000..5974e71071 --- /dev/null +++ b/ghost/tinybird/pipes/analytics_pages.pipe @@ -0,0 +1,24 @@ +NODE analytics_pages_1 +DESCRIPTION > + Aggregate by pathname and calculate session and hits + +SQL > + SELECT + site_uuid, + member_uuid, + member_status, + post_uuid, + toDate(timestamp) AS date, + device, + browser, + location, + pathname, + uniqState(session_id) AS visits, + countState() AS hits, + countStateIf(logged_in = true) AS logged_in_hits, + countStateIf(logged_in = false) AS logged_out_hits + FROM analytics_hits + GROUP BY date, device, browser, location, pathname, member_uuid, member_status, post_uuid, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_pages_mv diff --git a/ghost/tinybird/pipes/analytics_sessions.pipe b/ghost/tinybird/pipes/analytics_sessions.pipe new file mode 100644 index 0000000000..1eff0c3a7b --- /dev/null +++ b/ghost/tinybird/pipes/analytics_sessions.pipe @@ -0,0 +1,20 @@ +NODE analytics_sessions_1 +DESCRIPTION > + Aggregate by session_id and calculate session metrics + +SQL > + SELECT + site_uuid, + toDate(timestamp) AS date, + session_id, + anySimpleState(device) AS device, + anySimpleState(browser) AS browser, + anySimpleState(location) AS location, + minSimpleState(timestamp) AS first_hit, + maxSimpleState(timestamp) AS latest_hit, + countState() AS hits + FROM analytics_hits + GROUP BY date, session_id, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_sessions_mv diff --git a/ghost/tinybird/pipes/analytics_sources.pipe b/ghost/tinybird/pipes/analytics_sources.pipe new file mode 100644 index 0000000000..99a29e2f96 --- /dev/null +++ b/ghost/tinybird/pipes/analytics_sources.pipe @@ -0,0 +1,21 @@ +NODE analytics_sources_1 +DESCRIPTION > + Aggregate by referral and calculate session and hits + +SQL > + WITH (SELECT domainWithoutWWW(href) FROM analytics_hits LIMIT 1) AS current_domain + SELECT + site_uuid, + toDate(timestamp) AS date, + device, + browser, + location, + referrer, + uniqState(session_id) AS visits, + countState() AS hits + FROM analytics_hits + WHERE domainWithoutWWW(referrer) != current_domain + GROUP BY date, device, browser, location, referrer, site_uuid + +TYPE MATERIALIZED +DATASOURCE analytics_sources_mv diff --git a/ghost/tinybird/pipes/kpis.pipe b/ghost/tinybird/pipes/kpis.pipe new file mode 100644 index 0000000000..06a4f8e06a --- /dev/null +++ b/ghost/tinybird/pipes/kpis.pipe @@ -0,0 +1,130 @@ +DESCRIPTION > + Summary with general KPIs per date, including visits, page views, bounce rate and average session duration. + Accepts `date_from` and `date_to` date filter, all historical data if not passed. + Daily granularity, except when filtering one single day (hourly) + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the specified time range, so we call fill empty data points. + Filters "future" data points. + +SQL > + % + {% set _single_day = defined(date_from) and day_diff(date_from, date_to) == 0 %} + with + {% if defined(date_from) %} + toStartOfDay( + toDate( + {{ + Date( + date_from, + description="Starting day for filtering a date range", + required=False, + ) + }} + ) + ) as start, + {% else %} toStartOfDay(timestampAdd(today(), interval -7 day)) as start, + {% end %} + {% if defined(date_to) %} + toStartOfDay( + toDate( + {{ + Date( + date_to, + description="Finishing day for filtering a date range", + required=False, + ) + }} + ) + ) as end + {% else %} toStartOfDay(today()) as end + {% end %} + {% if _single_day %} + select + arrayJoin( + arrayMap( + x -> toDateTime(x), + range( + toUInt32(toDateTime(start)), toUInt32(timestampAdd(end, interval 1 day)), 3600 + ) + ) + ) as date + {% else %} + select + arrayJoin( + arrayMap( + x -> toDate(x), + range(toUInt32(start), toUInt32(timestampAdd(end, interval 1 day)), 24 * 3600) + ) + ) as date + {% end %} + where date <= now() + +NODE hits +DESCRIPTION > + Group by sessions and calculate metrics at that level + +SQL > + % + {% if defined(date_from) and day_diff(date_from, date_to) == 0 %} + select + site_uuid, + toStartOfHour(timestamp) as date, + session_id, + uniq(session_id) as visits, + count() as pageviews, + case when min(timestamp) = max(timestamp) then 1 else 0 end as is_bounce, + max(timestamp) as latest_hit_aux, + min(timestamp) as first_hit_aux + from analytics_hits + where toDate(timestamp) = {{ Date(date_from) }} + group by toStartOfHour(timestamp), session_id, site_uuid + {% else %} + select + site_uuid, + date, + session_id, + uniq(session_id) as visits, + countMerge(hits) as pageviews, + case when min(first_hit) = max(latest_hit) then 1 else 0 end as is_bounce, + max(latest_hit) as latest_hit_aux, + min(first_hit) as first_hit_aux + from analytics_sessions_mv + where + {% if defined(date_from) %} date >= {{ Date(date_from) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} and date <= {{ Date(date_to) }} + {% else %} and date <= today() + {% end %} + group by date, session_id, site_uuid + {% end %} + +NODE data +DESCRIPTION > + General KPIs per date, works for both summary metrics and trends charts. + +SQL > + select + site_uuid, + date, + uniq(session_id) as visits, + sum(pageviews) as pageviews, + sum(case when latest_hit_aux = first_hit_aux then 1 end) / visits as bounce_rate, + avg(latest_hit_aux - first_hit_aux) as avg_session_sec + from hits + group by date, site_uuid + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics + +SQL > +% + select a.date, b.visits, b.pageviews, b.bounce_rate, b.avg_session_sec + from timeseries a + left join data b using date + where site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} diff --git a/ghost/tinybird/pipes/top_browsers.pipe b/ghost/tinybird/pipes/top_browsers.pipe new file mode 100644 index 0000000000..5789296d84 --- /dev/null +++ b/ghost/tinybird/pipes/top_browsers.pipe @@ -0,0 +1,32 @@ +DESCRIPTION > + Top Browsers ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by browser and calculate hits and visits + +SQL > + % + select browser, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by browser + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_devices.pipe b/ghost/tinybird/pipes/top_devices.pipe new file mode 100644 index 0000000000..1988bac9ac --- /dev/null +++ b/ghost/tinybird/pipes/top_devices.pipe @@ -0,0 +1,33 @@ + +DESCRIPTION > + Top Device Types ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by device and calculate hits and visits + +SQL > + % + select device, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by device + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_locations.pipe b/ghost/tinybird/pipes/top_locations.pipe new file mode 100644 index 0000000000..61ad02aeb8 --- /dev/null +++ b/ghost/tinybird/pipes/top_locations.pipe @@ -0,0 +1,32 @@ +DESCRIPTION > + Top visiting Countries ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calculate hits and visits + +SQL > + % + select location, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_pages_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by location + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_pages.pipe b/ghost/tinybird/pipes/top_pages.pipe new file mode 100644 index 0000000000..41b65664fa --- /dev/null +++ b/ghost/tinybird/pipes/top_pages.pipe @@ -0,0 +1,38 @@ +DESCRIPTION > + Most visited pages for a given period. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by pagepath and calculate hits and visits + +SQL > + % + select + pathname, + uniqMerge(visits) as visits, + countMerge(hits) as hits, + countMerge(logged_in_hits) as logged_in_hits, + countMerge(logged_out_hits) as logged_out_hits + from analytics_pages_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + + group by pathname + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/top_sources.pipe b/ghost/tinybird/pipes/top_sources.pipe new file mode 100644 index 0000000000..1282fd70cc --- /dev/null +++ b/ghost/tinybird/pipes/top_sources.pipe @@ -0,0 +1,33 @@ + +DESCRIPTION > + Top traffic sources (domains), ordered by most visits. + Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. + Also `skip` and `limit` parameters for pagination. + +TOKEN "dashboard" READ + +NODE endpoint +DESCRIPTION > + Group by referral and calculate hits and visits + +SQL > + % + select domainWithoutWWW(referrer) as referrer, uniqMerge(visits) as visits, countMerge(hits) as hits + from analytics_sources_mv + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + {% if defined(date_from) %} + date + >= + {{ Date(date_from, description="Starting day for filtering a date range", required=False) }} + {% else %} date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date + <= + {{ Date(date_to, description="Finishing day for filtering a date range", required=False) }} + {% else %} and date <= today() + {% end %} + group by referrer + order by visits desc + limit {{ Int32(skip, 0) }},{{ Int32(limit, 50) }} diff --git a/ghost/tinybird/pipes/trend.pipe b/ghost/tinybird/pipes/trend.pipe new file mode 100644 index 0000000000..edbc74d79e --- /dev/null +++ b/ghost/tinybird/pipes/trend.pipe @@ -0,0 +1,36 @@ + +DESCRIPTION > + Visits trend over time for the last 30 minutes, filling the blanks. + Works great for the realtime chart. + +TOKEN "dashboard" READ + +NODE timeseries +DESCRIPTION > + Generate a timeseries for the last 30 minutes, so we call fill empty data points + +SQL > + with (now() - interval 30 minute) as start + select addMinutes(toStartOfMinute(start), number) as t + from (select arrayJoin(range(1, 31)) as number) + +NODE hits +DESCRIPTION > + Get last 30 minutes metrics gropued by minute + +SQL > + % + select toStartOfMinute(timestamp) as t, uniq(session_id) as visits + from analytics_hits + where + site_uuid = {{String(site_uuid, 'mock_site_uuid', description="Tenant ID", required=True)}} and + timestamp >= (now() - interval 30 minute) + group by toStartOfMinute(timestamp) + order by toStartOfMinute(timestamp) + +NODE endpoint +DESCRIPTION > + Join and generate timeseries with metrics for the last 30 minutes + +SQL > + select a.t, b.visits from timeseries a left join hits b on a.t = b.t order by a.t From 0720bc2bdb8f9084437481b512b4f518b0d11eac Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Fri, 30 Aug 2024 11:23:27 +0100 Subject: [PATCH 09/10] Updated multi-tenant param name for stats charts ref https://linear.app/tryghost/issue/ANAL-27/setup-tinybird-project-and-cicd - We've settled on a name for the param of site_uuid - Need to update the chart code that was added prior to this final decision --- ghost/admin/app/components/stats/charts/top-locations.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ghost/admin/app/components/stats/charts/top-locations.js b/ghost/admin/app/components/stats/charts/top-locations.js index 66d0d91369..11b0d6d8ef 100644 --- a/ghost/admin/app/components/stats/charts/top-locations.js +++ b/ghost/admin/app/components/stats/charts/top-locations.js @@ -15,14 +15,14 @@ export default class TopPages extends Component { /** * @typedef {Object} Params - * @property {string} cid + * @property {string} site_uuid * @property {string} [date_from] * @property {string} [date_to] * @property {number} [limit] * @property {number} [skip] */ const params = { - cid: this.config.stats.id, + site_uuid: this.config.stats.id, date_from: startDate.format('YYYY-MM-DD'), date_to: endDate.format('YYYY-MM-DD') }; From bf4e6600a9ce13bddb48361997b0a78719155395 Mon Sep 17 00:00:00 2001 From: Ghost CI <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 30 Aug 2024 15:05:40 +0000 Subject: [PATCH 10/10] v5.91.0 --- ghost/admin/package.json | 4 ++-- ghost/core/package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ghost/admin/package.json b/ghost/admin/package.json index 680f0118e4..3bff696f09 100644 --- a/ghost/admin/package.json +++ b/ghost/admin/package.json @@ -1,6 +1,6 @@ { "name": "ghost-admin", - "version": "5.90.2", + "version": "5.91.0", "description": "Ember.js admin client for Ghost", "author": "Ghost Foundation", "homepage": "http://ghost.org", @@ -207,4 +207,4 @@ } } } -} +} \ No newline at end of file diff --git a/ghost/core/package.json b/ghost/core/package.json index bd11b9f4c4..862b40460f 100644 --- a/ghost/core/package.json +++ b/ghost/core/package.json @@ -1,6 +1,6 @@ { "name": "ghost", - "version": "5.90.2", + "version": "5.91.0", "description": "The professional publishing platform", "author": "Ghost Foundation", "homepage": "https://ghost.org",