diff --git a/.gitignore b/.gitignore index 7090507371..b3e179754d 100644 --- a/.gitignore +++ b/.gitignore @@ -167,3 +167,8 @@ tsconfig.tsbuildinfo /apps/admin-x-settings/test-results/ /apps/admin-x-settings/playwright-report/ /apps/admin-x-settings/playwright/.cache/ + +# Tinybird +.tinyb +.venv +.diff_tmp diff --git a/ghost/tinybird/.tinyenv b/ghost/tinybird/.tinyenv new file mode 100644 index 0000000000..3281ddfae8 --- /dev/null +++ b/ghost/tinybird/.tinyenv @@ -0,0 +1,31 @@ + +# VERSION format is major.minor.patch-post where major, minor, patch and post are integer numbers +# bump post to deploy to the current live Release, rollback to previous post version is not available +# bump patch or minor to deploy a new Release and auto-promote it to live. Add TB_AUTO_PROMOTE=0 to create the Release in preview status +# bump major to deploy a new Release in preview status +VERSION=0.0.0 + + + +########## +# OPTIONAL env vars +# Deploy a new Release in preview status (default is 1) +# TB_AUTO_PROMOTE=0 + +# Check if deploy requires backfilling on preview (default is 1) +# TB_CHECK_BACKFILL_REQUIRED=0 + +# Force old Releases deletion on promote (default is 0) +# Setting it to 1 will remove oldest rollback Releases even when some resource is still in use +# TB_FORCE_REMOVE_OLDEST_ROLLBACK=0 + +# Don't print CLI version warning message if there's a new available version +# TB_VERSION_WARNING=0 + +# Skip regression tests +# TB_SKIP_REGRESSION=0 + +# Use `OBFUSCATE_REGEX_PATTERN` and `OBFUSCATE_PATTERN_SEPARATOR` environment variables to define a regex pattern and a separator (in case of a single string with multiple regex) to obfuscate secrets in the CLI output. +# OBFUSCATE_REGEX_PATTERN="https://(www\.)?[^/]+||^Follow these instructions =>" +# OBFUSCATE_PATTERN_SEPARATOR=|| +########## diff --git a/ghost/tinybird/README.md b/ghost/tinybird/README.md new file mode 100644 index 0000000000..d3230a50d6 --- /dev/null +++ b/ghost/tinybird/README.md @@ -0,0 +1,18 @@ +# Tinybird + +This folder contains configuration for Tinybird, so that the stats feature can be used. + +We sync this configuration with Tinybird via the Tinybird CLI. + +## Tinybird CLI + +The Tinybird CLI is used via Docker. + +```bash +yarn tb +``` + +Documentation for the Tinybird CLI: https://docs.tinybird.co/v/0.22.0/cli/overview +Note: you can use python if you prefer, but we use Docker for consistency. + +How to work with version control: https://docs.tinybird.co/v/0.22.0/guides/version-control diff --git a/ghost/tinybird/requirements.txt b/ghost/tinybird/requirements.txt new file mode 100644 index 0000000000..86035df960 --- /dev/null +++ b/ghost/tinybird/requirements.txt @@ -0,0 +1 @@ +tinybird-cli>=4,<5 \ No newline at end of file diff --git a/ghost/tinybird/scripts/append_fixtures.sh b/ghost/tinybird/scripts/append_fixtures.sh new file mode 100755 index 0000000000..e8745565af --- /dev/null +++ b/ghost/tinybird/scripts/append_fixtures.sh @@ -0,0 +1,21 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +directory="datasources/fixtures" +extensions=("csv" "ndjson") + +absolute_directory=$(realpath "$directory") + +for extension in "${extensions[@]}"; do + file_list=$(find "$absolute_directory" -type f -name "*.$extension") + + for file_path in $file_list; do + file_name=$(basename "$file_path") + file_name_without_extension="${file_name%.*}" + + command="tb datasource append $file_name_without_extension datasources/fixtures/$file_name" + echo $command + $command + done +done diff --git a/ghost/tinybird/scripts/exec_test.sh b/ghost/tinybird/scripts/exec_test.sh new file mode 100755 index 0000000000..50571d952d --- /dev/null +++ b/ghost/tinybird/scripts/exec_test.sh @@ -0,0 +1,58 @@ + +#!/usr/bin/env bash +set -euxo pipefail + +export TB_VERSION_WARNING=0 + +run_test() { + t=$1 + echo "** Running $t **" + echo "** $(cat $t)" + tmpfile=$(mktemp) + retries=0 + TOTAL_RETRIES=3 + + # When appending fixtures, we need to retry in case of the data is not replicated in time + while [ $retries -lt $TOTAL_RETRIES ]; do + # Run the test and store the output in a temporary file + bash $t $2 >$tmpfile + exit_code=$? + if [ "$exit_code" -eq 0 ]; then + # If the test passed, break the loop + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + break + # If the test failed, increment the retries counter and try again + else + retries=$((retries+1)) + fi + # If the bash command failed, print an error message and break the loop + else + break + fi + done + + if diff -B ${t}.result $tmpfile >/dev/null 2>&1; then + echo "✅ Test $t passed" + rm $tmpfile + return 0 + elif [ $retries -eq $TOTAL_RETRIES ]; then + echo "🚨 ERROR: Test $t failed, diff:"; + diff -B ${t}.result $tmpfile + rm $tmpfile + return 1 + else + echo "🚨 ERROR: Test $t failed with bash command exit code $?" + cat $tmpfile + rm $tmpfile + return 1 + fi + echo "" +} +export -f run_test + +fail=0 +find ./tests -name "*.test" -print0 | xargs -0 -I {} -P 4 bash -c 'run_test "$@"' _ {} || fail=1 + +if [ $fail == 1 ]; then + exit -1; +fi diff --git a/package.json b/package.json index b50208c85b..8a8034adfe 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "main": "yarn main:monorepo && yarn main:submodules", "main:monorepo": "git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn", "main:submodules": "git submodule sync && git submodule update && git submodule foreach \"git checkout main && git pull ${GHOST_UPSTREAM:-origin} main && yarn\"", - "prepare": "husky install .github/hooks" + "prepare": "husky install .github/hooks", + "tb": "docker run --rm -v $(pwd):/ghost -w /ghost/ghost/tinybird -it tinybirdco/tinybird-cli-docker" }, "resolutions": { "@tryghost/errors": "1.3.5",