Added posts exporter implementation (#16467)

fixes https://github.com/TryGhost/Team/issues/2779
fixes https://github.com/TryGhost/Team/issues/2781

Needs some more manual testing, unit tests and E2E tests. But the
exporting is implemented and some columns are removed based on the site
settings.

ghost/core/core/server/services/posts/posts-service.js

@@ -1,4 +1,5 @@
-const {PostsService} = require('@tryghost/posts-service');
+const {PostsService, PostsExporter} = require('@tryghost/posts-service');
+const url = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
 
 /**
  * @returns {InstanceType<PostsService>} instance of the PostsService
@@ -9,15 +10,33 @@ const getPostServiceInstance = () => {
     const models = require('../../models');
     const PostStats = require('./stats/post-stats');
     const emailService = require('../email-service');
+    const settingsCache = require('../../../shared/settings-cache');
+    const settingsHelpers = require('../settings-helpers');
 
     const postStats = new PostStats();
 
+    const postsExporter = new PostsExporter({
+        models: {
+            Post: models.Post,
+            Newsletter: models.Newsletter,
+            Label: models.Label
+        },
+        getPostUrl(post) {
+            const jsonModel = post.toJSON();
+            url.forPost(post.id, jsonModel, {options: {}});
+            return jsonModel.url;
+        },
+        settingsCache,
+        settingsHelpers
+    });
+
     return new PostsService({
         urlUtils: urlUtils,
         models: models,
         isSet: flag => labs.isSet(flag), // don't use bind, that breaks test subbing of labs
         stats: postStats,
-        emailService: emailService.service
+        emailService: emailService.service,
+        postsExporter
     });
 };
 

ghost/posts-service/index.js

@@ -1,3 +1,4 @@
 module.exports = {
-    PostsService: require('./lib/PostsService')
+    PostsService: require('./lib/PostsService'),
+    PostsExporter: require('./lib/PostsExporter')
 };

ghost/posts-service/lib/PostsExporter.js

@@ -0,0 +1,267 @@
+const nql = require('@tryghost/nql');
+const logging = require('@tryghost/logging');
+
+class PostsExporter {
+    #models;
+    #getPostUrl;
+    #settingsCache;
+    #settingsHelpers;
+
+    /**
+     * @param {Object} dependencies
+     * @param {Object} dependencies.models
+     * @param {Object} dependencies.models.Post
+     * @param {Object} dependencies.models.Newsletter
+     * @param {Object} dependencies.models.Label
+     * @param {Object} dependencies.getPostUrl
+     * @param {Object} dependencies.settingsCache
+     * @param {Object} dependencies.settingsHelpers
+     */
+    constructor({models, getPostUrl, settingsCache, settingsHelpers}) {
+        this.#models = models;
+        this.#getPostUrl = getPostUrl;
+        this.#settingsCache = settingsCache;
+        this.#settingsHelpers = settingsHelpers;
+    }
+
+    /**
+     *
+     * @param {object} options
+     * @param {string} [options.filter]
+     * @param {string} [options.order]
+     * @param {string|number} [options.limit]
+     */
+    async export({filter, order, limit}) {
+        const posts = await this.#models.Post.findPage({
+            filter,
+            order,
+            limit,
+            withRelated: [
+                'tiers',
+                'tags',
+                'authors',
+                'count.signups',
+                'count.paid_conversions',
+                'count.clicks',
+                'count.positive_feedback',
+                'count.negative_feedback',
+                'email'
+            ]
+        });
+
+        const newsletters = (await this.#models.Newsletter.findAll()).models;
+        const labels = (await this.#models.Label.findAll()).models;
+
+        const membersEnabled = this.#settingsHelpers.isMembersEnabled();
+        const membersTrackSources = membersEnabled && this.#settingsCache.get('members_track_sources');
+        const paidMembersEnabled = membersEnabled && this.#settingsHelpers.arePaidMembersEnabled();
+        const trackOpens = this.#settingsCache.get('email_track_opens');
+        const trackClicks = this.#settingsCache.get('email_track_clicks');
+        const hasNewslettersWithFeedback = !!newsletters.find(newsletter => newsletter.get('feedback_enabled'));
+
+        const mapped = posts.data.map((post) => {
+            let email = post.related('email');
+            let published = true;
+            if (post.get('status') === 'draft' || post.get('status') === 'scheduled') {
+                // Manually clear it to avoid including information for a post that was reverted to draft
+                email = null;
+                published = false;
+            }
+
+            const feedbackEnabled = email && email.get('feedback_enabled') && hasNewslettersWithFeedback;
+            const showEmailClickAnalytics = trackClicks && email && email.get('track_clicks');
+
+            return {
+                title: post.get('title'),
+                url: this.#getPostUrl(post),
+                author: post.related('authors').map(author => author.get('name')).join(', '),
+                status: this.mapPostStatus(post.get('status'), !!email),
+                created_at: post.get('created_at'),
+                updated_at: post.get('updated_at'),
+                published_at: published ? post.get('published_at') : null,
+                featured: post.get('featured'),
+                tags: post.related('tags').map(tag => tag.get('name')).join(', '),
+                post_access: this.postAccessToString(post),
+                email_recipients: email ? this.humanReadableEmailRecipientFilter(email?.get('recipient_filter'), labels) : null,
+                newsletter: newsletters.length > 1 && post.get('newsletter_id') && email ? newsletters.find(newsletter => newsletter.get('id') === post.get('newsletter_id')).get('name') : null,
+                sends: email?.get('email_count') ?? null,
+                opens: trackOpens ? (email?.get('opened_count') ?? null) : null,
+                clicks: showEmailClickAnalytics ? (post.get('count__clicks') ?? 0) : null,
+                free_signups: membersTrackSources && published ? (post.get('count__signups') ?? 0) : null,
+                paid_signups: membersTrackSources && paidMembersEnabled && published ? (post.get('count__paid_conversions') ?? 0) : null,
+                reacted_with_more_like_this: feedbackEnabled ? (post.get('count__positive_feedback') ?? 0) : null,
+                reacted_with_less_like_this: feedbackEnabled ? (post.get('count__negative_feedback') ?? 0) : null
+            };
+        });
+
+        if (mapped.length) {
+            // Limit the amount of removeable columns so the structure is consistent depending on global settings
+            const removeableColumns = [];
+
+            if (newsletters.length <= 1) {
+                removeableColumns.push('newsletter');
+            }
+
+            if (!membersEnabled) {
+                removeableColumns.push('email_recipients', 'sends', 'opens', 'clicks', 'reacted_with_more_like_this', 'reacted_with_less_like_this');
+            } else if (!hasNewslettersWithFeedback) {
+                removeableColumns.push('reacted_with_more_like_this', 'reacted_with_less_like_this');
+            }
+
+            if (!membersEnabled && !trackClicks) {
+                removeableColumns.push('clicks');
+            }
+
+            if (!membersEnabled && !trackOpens) {
+                removeableColumns.push('opens');
+            }
+
+            if (!membersTrackSources || !membersEnabled) {
+                removeableColumns.push('free_signups', 'paid_signups');
+            } else if (!paidMembersEnabled) {
+                removeableColumns.push('paid_signups');
+            }
+
+            // Note the strict null check: we allow columns that are all zero
+            const columnsToRemove = removeableColumns.filter(key => mapped.every(row => !row[key] && row[key] !== 0));
+
+            for (const columnToRemove of columnsToRemove) {
+                for (const row of mapped) {
+                    delete row[columnToRemove];
+                }
+            }
+        }
+
+        return mapped;
+    }
+
+    mapPostStatus(status, hasEmail) {
+        if (status === 'draft') {
+            return 'draft';
+        }
+
+        if (status === 'scheduled') {
+            return 'scheduled';
+        }
+
+        if (status === 'sent') {
+            return 'emailed only';
+        }
+
+        if (status === 'published') {
+            if (hasEmail) {
+                return 'published and emailed';
+            }
+            return 'published only';
+        }
+        return status;
+    }
+
+    postAccessToString(post) {
+        const visibility = post.get('visibility');
+        if (visibility === 'public') {
+            return 'Public';
+        }
+
+        if (visibility === 'members') {
+            return 'Free members';
+        }
+
+        if (visibility === 'paid') {
+            return 'Paid members';
+        }
+
+        if (visibility === 'tiers') {
+            const tiers = post.related('tiers');
+            if (tiers.length === 0) {
+                return 'Nobody';
+            }
+
+            return tiers.map(tier => tier.get('name')).join(', ');
+        }
+
+        return visibility;
+    }
+
+    /**
+     * @private Convert an email filter to a human readable string
+     * @param {string} recipientFilter
+     * @param {*} allLabels
+     * @returns
+     */
+    humanReadableEmailRecipientFilter(recipientFilter, allLabels) {
+        // Examples: "label:test"; "label:test,label:batch1"; "status:-free,label:test", "all"
+        if (recipientFilter === 'all') {
+            return 'all';
+        }
+
+        try {
+            const parsed = nql(recipientFilter).parse();
+            const strings = this.filterToString(parsed, allLabels);
+            return strings.join(', ');
+        } catch (e) {
+            logging.error(e);
+            return recipientFilter;
+        }
+    }
+
+    /**
+     * @private Convert an email filter to a human readable string
+     * @param {*} filter Parsed NQL filter
+     * @param {*} allLabels All available member labels
+     * @returns
+     */
+    filterToString(filter, allLabels) {
+        if (!filter) {
+            return [];
+        }
+        const strings = [];
+        if (filter.$and) {
+            // Not supported
+        } else if (filter.$or) {
+            for (const subfilter of filter.$or) {
+                strings.push(...this.filterToString(subfilter, allLabels));
+            }
+        } else if (filter.yg) {
+            // Single filter grouped in brackets
+            strings.push(...this.filterToString(filter.yg, allLabels));
+        } else {
+            for (const key of Object.keys(filter)) {
+                if (key === 'label') {
+                    if (typeof filter.label === 'string') {
+                        const labelSlug = filter.label;
+                        const label = allLabels.find(l => l.get('slug') === labelSlug);
+                        if (label) {
+                            strings.push(label.get('name'));
+                        } else {
+                            strings.push(labelSlug);
+                        }
+                    }
+                }
+                if (key === 'status') {
+                    if (typeof filter.status === 'string') {
+                        if (filter.status === 'free') {
+                            strings.push('free members');
+                        } else if (filter.status === 'paid') {
+                            strings.push('paid members');
+                        } else if (filter.status === 'comped') {
+                            strings.push('comped members');
+                        }
+                    } else {
+                        if (filter.status.$ne === 'free') {
+                            strings.push('paid members');
+                        }
+
+                        if (filter.status.$ne === 'paid') {
+                            strings.push('free members');
+                        }
+                    }
+                }
+            }
+        }
+
+        return strings;
+    }
+}
+
+module.exports = PostsExporter;

ghost/posts-service/lib/PostsService.js

@@ -8,12 +8,13 @@ const messages = {
 };
 
 class PostsService {
-    constructor({urlUtils, models, isSet, stats, emailService}) {
+    constructor({urlUtils, models, isSet, stats, emailService, postsExporter}) {
         this.urlUtils = urlUtils;
         this.models = models;
         this.isSet = isSet;
         this.stats = stats;
         this.emailService = emailService;
+        this.postsExporter = postsExporter;
     }
 
     async editPost(frame) {
@@ -57,16 +58,8 @@ class PostsService {
         return model;
     }
 
-    async export() {
+    async export(frame) {
-        // Placeholder implementation
+        return await this.postsExporter.export(frame.options);
-        return [
-            {
-                title: 'Example',
-                url: 'https://example.com',
-                author: 'Jamie Larson',
-                status: 'published'
-            }
-        ];
     }
 
     async getProductsFromVisibilityFilter(visibilityFilter) {