Updated source grouping to use known referrers data

refs TryGhost/Team#1923

- updates referrer translator logic to use grouping for known domains/urls to calculate source and medium
- returns first external ref url if nothing matches in history in first pass

ghost/member-attribution/lib/referrer-translator.js

@@ -5,6 +5,8 @@
  * @prop {URL|null} [refUrl]
  */
 
+const knownReferrers = require('@tryghost/referrers');
+
 /**
  * Translates referrer info into Source and Medium
  */
@@ -64,7 +66,7 @@ class ReferrerTranslator {
 
             // If referrer is from query params
             if (refSource) {
-                const urlData = this.getDataFromUrl() || {};
+                const urlData = refUrl ? this.getDataFromUrl(refUrl) : null;
                 return {
                     refSource: refSource,
                     refMedium: refMedium || urlData?.medium || null,
@@ -73,9 +75,8 @@ class ReferrerTranslator {
             }
 
             // If referrer is known external URL
-            // TODO: Use list of known external urls to fetch source/medium
             if (refUrl && !this.isSiteDomain(refUrl)) {
-                const urlData = this.getDataFromUrl();
+                const urlData = this.getDataFromUrl(refUrl);
 
                 if (urlData) {
                     return {
@@ -87,13 +88,34 @@ class ReferrerTranslator {
             }
         }
 
+        // Return any referrer URL in history that is not site domain
+        for (const item of history) {
+            const refUrl = this.getUrlFromStr(item.refUrl);
+
+            if (refUrl && !this.isSiteDomain(refUrl)) {
+                return {
+                    refSource: null,
+                    refMedium: null,
+                    refUrl: refUrl
+                };
+            }
+        }
+
         return null;
     }
 
     // Fetches referrer data from known external URLs
-    //TODO: Use list of known external urls to fetch source/medium
-    getDataFromUrl() {
-        return null;
+    getDataFromUrl(url) {
+        // Allow matching both "google.ac/products" and "google.ac" as a source
+        const urlHostPath = url?.host + url?.pathname;
+        const urlDataKey = Object.keys(knownReferrers).sort((a, b) => {
+            // The longer key has higher the priority so google.ac/products is selected before google.ac
+            return b.length - a.length;
+        }).find((source) => {
+            return urlHostPath?.startsWith(source);
+        });
+
+        return urlDataKey ? knownReferrers[urlDataKey] : null;
     }
 
     /**

ghost/member-attribution/package.json

@@ -21,6 +21,7 @@
   },
   "dependencies": {
     "@tryghost/domain-events": "0.0.0",
-    "@tryghost/member-events": "0.0.0"
+    "@tryghost/member-events": "0.0.0",
+    "@tryghost/referrers": "0.0.0"
   }
 }

ghost/member-attribution/test/referrer-translator.test.js

@@ -163,6 +163,80 @@ describe('ReferrerTranslator', function () {
             });
         });
 
+        describe('returns source and medium for', function () {
+            it('known external url with path', async function () {
+                should(translator.getReferrerDetails([
+                    {
+                        refSource: null,
+                        refMedium: null,
+                        refUrl: 'https://google.ac/products'
+                    },
+                    {
+                        refSource: null,
+                        refMedium: null,
+                        refUrl: 'https://t.co/'
+                    },
+                    {
+                        refSource: 'publisher-weekly-newsletter',
+                        refMedium: null,
+                        refUrl: null
+                    },
+                    {
+                        refSource: 'ghost-explore',
+                        refMedium: null,
+                        refUrl: null
+                    }
+                ])).eql({
+                    refSource: 'Google Product Search',
+                    refMedium: 'search',
+                    refUrl: new URL('https://google.ac/products')
+                });
+            });
+
+            it('known external url without path', async function () {
+                should(translator.getReferrerDetails([
+                    {
+                        refSource: null,
+                        refMedium: null,
+                        refUrl: 'https://t.co/'
+                    },
+                    {
+                        refSource: 'publisher-weekly-newsletter',
+                        refMedium: null,
+                        refUrl: null
+                    },
+                    {
+                        refSource: 'ghost-explore',
+                        refMedium: null,
+                        refUrl: null
+                    }
+                ])).eql({
+                    refSource: 'Twitter',
+                    refMedium: 'social',
+                    refUrl: new URL('https://t.co/')
+                });
+            });
+        });
+
+        it('returns external ref url if nothing matches', async function () {
+            should(translator.getReferrerDetails([
+                {
+                    refSource: null,
+                    refMedium: null,
+                    refUrl: 'https://example.com'
+                },
+                {
+                    refSource: null,
+                    refMedium: null,
+                    refUrl: 'https://sample.com'
+                }
+            ])).eql({
+                refSource: null,
+                refMedium: null,
+                refUrl: new URL('https://sample.com')
+            });
+        });
+
         it('returns null for empty history', async function () {
             should(translator.getReferrerDetails([])).eql(null);
         });