feat: ✨ return HTTP 410 page for expired notes

server/package-lock.json

@@ -10,6 +10,7 @@
       "license": "MIT",
       "dependencies": {
         "@prisma/client": "^4.0.0",
+        "bloom-filters": "^3.0.0",
         "body-parser": "^1.20.0",
         "class-validator": "^0.13.2",
         "dotenv": "^16.0.1",
@@ -640,6 +641,14 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
+    "node_modules/base64-arraybuffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz",
+      "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
     "node_modules/binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
@@ -649,6 +658,25 @@
         "node": ">=8"
       }
     },
+    "node_modules/bloom-filters": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bloom-filters/-/bloom-filters-3.0.0.tgz",
+      "integrity": "sha512-DBDgLkYokKS5NA5y8P9fuTavKQCkleAP39yqpW/5Nab/vwzHv+wOPRM/yDAStghARDleyRI4orW91uuxj48LKQ==",
+      "dependencies": {
+        "base64-arraybuffer": "^1.0.2",
+        "is-buffer": "^2.0.5",
+        "lodash": "^4.17.15",
+        "lodash.eq": "^4.0.0",
+        "lodash.indexof": "^4.0.5",
+        "long": "^5.2.0",
+        "reflect-metadata": "^0.1.13",
+        "seedrandom": "^3.0.5",
+        "xxhashjs": "^0.2.2"
+      },
+      "engines": {
+        "node": ">=12"
+      }
+    },
     "node_modules/body-parser": {
       "version": "1.20.0",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
@@ -1018,6 +1046,11 @@
         "node": ">= 8"
       }
     },
+    "node_modules/cuint": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
+      "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
+    },
     "node_modules/debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -2357,6 +2390,28 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-buffer": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+      "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/feross"
+        },
+        {
+          "type": "patreon",
+          "url": "https://www.patreon.com/feross"
+        },
+        {
+          "type": "consulting",
+          "url": "https://feross.org/support"
+        }
+      ],
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/is-callable": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz",
@@ -2644,6 +2699,26 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
+    "node_modules/lodash.eq": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/lodash.eq/-/lodash.eq-4.0.0.tgz",
+      "integrity": "sha512-vbrJpXL6kQNG6TkInxX12DZRfuYVllSxhwYqjYB78g2zF3UI15nFO/0AgmZnZRnaQ38sZtjCiVjGr2rnKt4v0g=="
+    },
+    "node_modules/lodash.indexof": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/lodash.indexof/-/lodash.indexof-4.0.5.tgz",
+      "integrity": "sha512-t9wLWMQsawdVmf6/IcAgVGqAJkNzYVcn4BHYZKTPW//l7N5Oq7Bq138BaVk19agcsPZePcidSgTTw4NqS1nUAw=="
+    },
+    "node_modules/long": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz",
+      "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w=="
+    },
     "node_modules/loupe": {
       "version": "2.3.4",
       "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz",
@@ -3841,6 +3916,11 @@
         "tslib": "^1.9.3"
       }
     },
+    "node_modules/reflect-metadata": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
+      "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg=="
+    },
     "node_modules/regexp.prototype.flags": {
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",
@@ -3954,6 +4034,11 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "node_modules/seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
+    },
     "node_modules/semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -5039,6 +5124,14 @@
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
+    "node_modules/xxhashjs": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
+      "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
+      "dependencies": {
+        "cuint": "^0.2.2"
+      }
+    },
     "node_modules/y18n": {
       "version": "5.0.8",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",
@@ -5598,12 +5691,33 @@
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz",
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="
     },
+    "base64-arraybuffer": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz",
+      "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ=="
+    },
     "binary-extensions": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz",
       "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==",
       "dev": true
     },
+    "bloom-filters": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/bloom-filters/-/bloom-filters-3.0.0.tgz",
+      "integrity": "sha512-DBDgLkYokKS5NA5y8P9fuTavKQCkleAP39yqpW/5Nab/vwzHv+wOPRM/yDAStghARDleyRI4orW91uuxj48LKQ==",
+      "requires": {
+        "base64-arraybuffer": "^1.0.2",
+        "is-buffer": "^2.0.5",
+        "lodash": "^4.17.15",
+        "lodash.eq": "^4.0.0",
+        "lodash.indexof": "^4.0.5",
+        "long": "^5.2.0",
+        "reflect-metadata": "^0.1.13",
+        "seedrandom": "^3.0.5",
+        "xxhashjs": "^0.2.2"
+      }
+    },
     "body-parser": {
       "version": "1.20.0",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz",
@@ -5905,6 +6019,11 @@
         "which": "^2.0.1"
       }
     },
+    "cuint": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz",
+      "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw=="
+    },
     "debug": {
       "version": "2.6.9",
       "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -6826,6 +6945,11 @@
         "has-tostringtag": "^1.0.0"
       }
     },
+    "is-buffer": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz",
+      "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ=="
+    },
     "is-callable": {
       "version": "1.2.4",
       "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz",
@@ -7020,6 +7144,26 @@
         "p-locate": "^5.0.0"
       }
     },
+    "lodash": {
+      "version": "4.17.21",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz",
+      "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="
+    },
+    "lodash.eq": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/lodash.eq/-/lodash.eq-4.0.0.tgz",
+      "integrity": "sha512-vbrJpXL6kQNG6TkInxX12DZRfuYVllSxhwYqjYB78g2zF3UI15nFO/0AgmZnZRnaQ38sZtjCiVjGr2rnKt4v0g=="
+    },
+    "lodash.indexof": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/lodash.indexof/-/lodash.indexof-4.0.5.tgz",
+      "integrity": "sha512-t9wLWMQsawdVmf6/IcAgVGqAJkNzYVcn4BHYZKTPW//l7N5Oq7Bq138BaVk19agcsPZePcidSgTTw4NqS1nUAw=="
+    },
+    "long": {
+      "version": "5.2.0",
+      "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz",
+      "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w=="
+    },
     "loupe": {
       "version": "2.3.4",
       "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz",
@@ -7907,6 +8051,11 @@
         "tslib": "^1.9.3"
       }
     },
+    "reflect-metadata": {
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz",
+      "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg=="
+    },
     "regexp.prototype.flags": {
       "version": "1.4.3",
       "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz",
@@ -7973,6 +8122,11 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "seedrandom": {
+      "version": "3.0.5",
+      "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz",
+      "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg=="
+    },
     "semver": {
       "version": "6.3.0",
       "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz",
@@ -8761,6 +8915,14 @@
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
       "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
     },
+    "xxhashjs": {
+      "version": "0.2.2",
+      "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz",
+      "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==",
+      "requires": {
+        "cuint": "^0.2.2"
+      }
+    },
     "y18n": {
       "version": "5.0.8",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz",

server/package.json

@@ -6,7 +6,7 @@
   "scripts": {
     "test": "run-s test:db:reset test:test",
     "coverage": "run-s test:db:reset test:coverage",
-    "test-watch": "dotenv -e .env.test -- vitest unit --coverage",
+    "test-watch": "dotenv -v UNIT_TEST=TRUE  -e .env.test -- vitest unit",
     "test:test": "dotenv -e .env.test -- vitest run --no-threads",
     "test:coverage": "dotenv -e .env.test -- vitest run --no-threads --coverage",
     "test:db:reset": "dotenv -e .env.test -- npx prisma migrate reset -f",
@@ -17,6 +17,7 @@
   "license": "MIT",
   "dependencies": {
     "@prisma/client": "^4.0.0",
+    "bloom-filters": "^3.0.0",
     "body-parser": "^1.20.0",
     "class-validator": "^0.13.2",
     "dotenv": "^16.0.1",

server/prisma/migrations/20220809194448_add_bloom_filter_table/migration.sql

@@ -0,0 +1,5 @@
+-- CreateTable
+CREATE TABLE "BloomFilter" (
+    "name" TEXT NOT NULL PRIMARY KEY,
+    "serializedFilter" BLOB NOT NULL
+);

server/prisma/schema.prisma

@@ -29,3 +29,8 @@ model event {
   error              String?
   expire_window_days Int?
 }
+
+model BloomFilter {
+  name             String @id
+  serializedFilter Bytes
+}

server/src/app.integration.test.ts

@@ -2,8 +2,10 @@ import { app } from "./app";
 import supertest from "supertest";
 import { describe, it, expect } from "vitest";
 import prisma from "./db/client";
-import { cleanExpiredNotes } from "./tasks/deleteExpiredNotes";
+import { deleteExpiredNotes } from "./tasks/deleteExpiredNotes";
 import { EventType } from "./logging/EventLogger";
+import { getFilter } from "./db/bloomFilter.dao";
+import { getExpiredNoteFilter } from "./lib/expiredNoteFilter";
 
 // const testNote with base64 ciphertext and hmac
 const testNote = {
@@ -182,12 +184,12 @@ describe("Clean expired notes", () => {
     expect(res.statusCode).toBe(200);
 
     // run cleanup
-    const nDeleted = await cleanExpiredNotes();
+    const nDeleted = await deleteExpiredNotes();
     expect(nDeleted).toBeGreaterThan(0);
 
-    // make sure note is gone
+    // if the note is added to the expire filter, it returns 410
     res = await supertest(app).get(`/api/note/${id}`);
-    expect(res.statusCode).toBe(404);
+    expect(res.statusCode).toBe(410);
 
     // sleep 100ms to allow all events to be logged
     await new Promise((resolve) => setTimeout(resolve, 200));

server/src/app.ts

@@ -4,7 +4,7 @@ import helmet from "helmet";
 import pinoHttp from "pino-http";
 import logger from "./logging/logger";
 import { notesRoute } from "./controllers/note/note.router";
-import { cleanExpiredNotes, cleanInterval } from "./tasks/deleteExpiredNotes";
+import { deleteExpiredNotes, deleteInterval } from "./tasks/deleteExpiredNotes";
 
 // Initialize middleware clients
 export const app: Express = express();
@@ -32,4 +32,4 @@ app.use(
 app.use("/api/note/", notesRoute);
 
 // Run periodic tasks
-setInterval(cleanExpiredNotes, cleanInterval);
+setInterval(deleteExpiredNotes, deleteInterval);

server/src/controllers/note/note.get.controller.ts

@@ -1,8 +1,8 @@
 import { NextFunction, Request, Response } from "express";
+import { getExpiredNoteFilter } from "../../lib/expiredNoteFilter";
 import EventLogger from "../../logging/EventLogger";
 import { getConnectingIp } from "../../util";
 import { getNote } from "./note.dao";
-
 export async function getNoteController(
   req: Request,
   res: Response,
@@ -20,13 +20,25 @@ export async function getNoteController(
         });
         res.send(note);
       } else {
-        await EventLogger.readEvent({
+        // check the expired filter to see if the note was expired
-          success: false,
+        const expiredFilter = await getExpiredNoteFilter();
-          host: ip,
+        if (expiredFilter.hasNoteId(req.params.id)) {
-          note_id: req.params.id,
+          await EventLogger.readEvent({
-          error: "Note not found",
+            success: false,
-        });
+            host: ip,
-        res.status(404).send();
+            note_id: req.params.id,
+            error: "Note expired",
+          });
+          res.status(410).send("Note expired");
+        } else {
+          await EventLogger.readEvent({
+            success: false,
+            host: ip,
+            note_id: req.params.id,
+            error: "Note not found",
+          });
+          res.status(404).send();
+        }
       }
     })
     .catch(async (err) => {

server/src/db/bloomFilter.dao.integration.test.ts

@@ -0,0 +1,45 @@
+import { ScalableBloomFilter } from "bloom-filters";
+import { describe, it, expect } from "vitest";
+import { getFilter, upsertFilter } from "./bloomFilter.dao";
+
+describe("Serialization and deserialization of filters", () => {
+  it("Should serialize and deserialize filters", async () => {
+    // Serialize a new filter
+    const testFilter = new ScalableBloomFilter();
+    testFilter.add("item0");
+    testFilter.add("item1");
+    testFilter.add("item2");
+    await upsertFilter("testFilter", testFilter);
+
+    // Deserialize the filter
+    const deserializedFilter = await getFilter<ScalableBloomFilter>(
+      "testFilter",
+      ScalableBloomFilter
+    );
+    expect(deserializedFilter.has("item0")).toBe(true);
+    expect(deserializedFilter.has("item1")).toBe(true);
+    expect(deserializedFilter.has("item2")).toBe(true);
+    expect(deserializedFilter.has("not_in_filter")).toBe(false);
+
+    // update the filter and serialize it
+    deserializedFilter.add("item3");
+    await upsertFilter("testFilter", deserializedFilter);
+
+    // Deserialize the filter again
+    const deserializedFilter2 = await getFilter<ScalableBloomFilter>(
+      "testFilter",
+      ScalableBloomFilter
+    );
+    expect(deserializedFilter2.has("item0")).toBe(true);
+    expect(deserializedFilter2.has("item1")).toBe(true);
+    expect(deserializedFilter2.has("item2")).toBe(true);
+    expect(deserializedFilter2.has("item3")).toBe(true);
+    expect(deserializedFilter2.has("not_in_filter")).toBe(false);
+  });
+
+  it("Should throw error if no filter is saved for given name", async () => {
+    await expect(
+      getFilter<ScalableBloomFilter>("not_existing", ScalableBloomFilter)
+    ).rejects.toThrow("No BloomFilter found");
+  });
+});

server/src/db/bloomFilter.dao.ts

@@ -0,0 +1,68 @@
+import { BaseFilter } from "bloom-filters";
+import prisma from "../db/client";
+
+interface IDeserializedFilter {
+  fromJSON: (json: JSON) => BaseFilter;
+}
+
+/**
+ *  Get a bloom filter from the database.
+ * @param name Name of the filter in database
+ * @param cls Class of the filter to deserialize
+ * @returns Deserialized filter of type T
+ * @throws Error if no filter is found in database
+ */
+export async function getFilter<T extends BaseFilter>(
+  name: string,
+  cls: IDeserializedFilter
+): Promise<T> {
+  const bloomFilter = await prisma.bloomFilter.findUniqueOrThrow({
+    where: {
+      name: name,
+    },
+  });
+  const serializedFilter = bloomFilter.serializedFilter;
+  return deserializeFilter<T>(serializedFilter, cls);
+}
+
+/**
+ * Creates a filter in the database if it does not exist yet.
+ * If it exists, it will be overwritten.
+ * @param name Name of the filter in database
+ * @param filter filter object to serialize
+ */
+export async function upsertFilter(
+  name: string,
+  filter: BaseFilter
+): Promise<void> {
+  const serializedFilter = serializeFilter(filter);
+  await prisma.bloomFilter.upsert({
+    where: {
+      name: name,
+    },
+    update: {
+      serializedFilter: serializedFilter,
+    },
+    create: {
+      name: name,
+      serializedFilter: serializedFilter,
+    },
+  });
+}
+
+function serializeFilter(filter: BaseFilter): Buffer {
+  const filterJSON = filter.saveAsJSON();
+  const filterString = JSON.stringify(filterJSON);
+  const filterBuffer = Buffer.from(filterString, "utf-8");
+  return filterBuffer;
+}
+
+function deserializeFilter<T extends BaseFilter>(
+  serializedFilter: Buffer,
+  cls: IDeserializedFilter
+): T {
+  const filterString = serializedFilter.toString("utf-8");
+  const filterJSON = JSON.parse(filterString);
+  const filter = cls.fromJSON(filterJSON) as T;
+  return filter;
+}

server/src/db/client.ts

@@ -1,5 +1,9 @@
 import { PrismaClient } from "@prisma/client";
 
+if (process.env.UNIT_TEST === "TRUE") {
+  throw Error("Database operations must be mocked in unit tests.");
+}
+
 const prisma = new PrismaClient();
 
 export default prisma;

server/src/lib/expiredNoteFilter.ts

@@ -0,0 +1,58 @@
+import { ScalableBloomFilter } from "bloom-filters";
+import { getFilter, upsertFilter } from "../db/bloomFilter.dao";
+
+export class ExpiredNoteFilter {
+  _filter: ScalableBloomFilter;
+  static FILTER_NAME = "expiredNotes";
+
+  private constructor(filter: ScalableBloomFilter) {
+    this._filter = filter;
+  }
+
+  public static async deserializeFromDb(): Promise<ExpiredNoteFilter> {
+    return ExpiredNoteFilter._deserializeFilter()
+      .catch((err) => {
+        if (err.message === "No BloomFilter found") {
+          return new ScalableBloomFilter();
+        } else {
+          throw err;
+        }
+      })
+      .then((filter) => {
+        return new ExpiredNoteFilter(filter);
+      });
+  }
+
+  public addNoteIds(noteIds: string[]): Promise<void> {
+    noteIds.forEach((noteId) => {
+      this._filter.add(noteId);
+    });
+    return this._serialize();
+  }
+
+  public hasNoteId(noteId: string): boolean {
+    return this._filter.has(noteId);
+  }
+
+  private _serialize(): Promise<void> {
+    return upsertFilter(ExpiredNoteFilter.FILTER_NAME, this._filter);
+  }
+
+  private static _deserializeFilter(): Promise<ScalableBloomFilter> {
+    return getFilter<ScalableBloomFilter>(
+      this.FILTER_NAME,
+      ScalableBloomFilter
+    );
+  }
+}
+
+let _filter: ExpiredNoteFilter;
+
+export async function getExpiredNoteFilter(): Promise<ExpiredNoteFilter> {
+  if (_filter) {
+    return _filter;
+  } else {
+    _filter = await ExpiredNoteFilter.deserializeFromDb();
+    return _filter;
+  }
+}

server/src/lib/expiredNoteFilter.unit.test.ts

@@ -0,0 +1,93 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { ExpiredNoteFilter } from "./expiredNoteFilter";
+import { ScalableBloomFilter } from "bloom-filters";
+
+import * as dao from "../db/bloomFilter.dao";
+vi.mock("../db/bloomFilter.dao", () => ({
+  getFilter: vi.fn(),
+  upsertFilter: vi.fn(),
+}));
+
+describe("Deserialization from database", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should create a new filter if no filter is found in database", async () => {
+    // mock no filter in database
+    const mockedDao = vi.mocked(dao);
+    mockedDao.getFilter.mockRejectedValue(new Error("No BloomFilter found"));
+
+    // test instatiation
+    const testFilter = await ExpiredNoteFilter.deserializeFromDb();
+    expect(mockedDao.getFilter).toHaveBeenCalledWith(
+      "expiredNotes",
+      ScalableBloomFilter
+    );
+    expect(testFilter).toBeInstanceOf(ExpiredNoteFilter);
+
+    // expect the _filter property to be a fresh ScalableBloomFilter (capacity 8)
+    expect(testFilter._filter).toBeInstanceOf(ScalableBloomFilter);
+    expect(testFilter._filter.capacity()).toBe(8);
+  });
+
+  it("should deserialize a filter from the database", async () => {
+    // mock prefilled bloom filter
+    const bloomFilter = new ScalableBloomFilter();
+    bloomFilter.add("test");
+    bloomFilter.add("test2");
+    const mockedDao = vi.mocked(dao);
+    mockedDao.getFilter.mockResolvedValue(bloomFilter);
+
+    // test instatiation
+    const testFilter = await ExpiredNoteFilter.deserializeFromDb();
+    expect(mockedDao.getFilter).toHaveBeenCalledWith(
+      "expiredNotes",
+      ScalableBloomFilter
+    );
+    expect(testFilter._filter).toBe(bloomFilter);
+
+    // expect the testFilter to have the same content as the mocked bloom filter
+    expect(testFilter.hasNoteId("test")).toBe(true);
+    expect(testFilter.hasNoteId("test2")).toBe(true);
+  });
+});
+
+describe("Filter operations and serialization", () => {
+  let testFilter: ExpiredNoteFilter;
+
+  beforeEach(async () => {
+    const mockedDao = vi.mocked(dao);
+    mockedDao.getFilter.mockResolvedValue(new ScalableBloomFilter());
+  });
+
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should add multiple noteIds to the filter", async () => {
+    testFilter = await ExpiredNoteFilter.deserializeFromDb();
+    testFilter.addNoteIds(["test", "test2"]);
+    expect(testFilter.hasNoteId("test")).toBe(true);
+    expect(testFilter.hasNoteId("test2")).toBe(true);
+
+    // expect the filter to be serialized to database
+    const mockedDao = vi.mocked(dao);
+    expect(mockedDao.upsertFilter).toHaveBeenCalledWith(
+      "expiredNotes",
+      testFilter._filter
+    );
+  });
+
+  it("Should have an error rate <1% for 1000 elements", async () => {
+    testFilter = await ExpiredNoteFilter.deserializeFromDb();
+    const elements = Array.from({ length: 1000 }, (_, i) => i.toString());
+    testFilter.addNoteIds(elements);
+
+    // expect the filter to have an error rate <1%
+    const errorRate = elements
+      .map<number>((el) => (testFilter.hasNoteId(el) ? 0 : 1))
+      .reduce((acc, cur) => acc + cur, 0);
+    expect(errorRate).toBeLessThan(10);
+  });
+});

server/src/logging/__mocks__/EventLogger.ts

@@ -0,0 +1,9 @@
+import { vi } from "vitest";
+
+const mockedEventLogger = {
+  writeEvent: vi.fn(),
+  readEvent: vi.fn(),
+  purgeEvent: vi.fn(),
+};
+
+export default mockedEventLogger;

server/src/tasks/deleteExpiredNotes.ts

@@ -1,12 +1,9 @@
 import { deleteNotes, getExpiredNotes } from "../controllers/note/note.dao";
+import { getExpiredNoteFilter } from "../lib/expiredNoteFilter";
 import EventLogger from "../logging/EventLogger";
 import logger from "../logging/logger";
 
-export const cleanInterval =
+export async function deleteExpiredNotes(): Promise<number> {
-  Math.max(parseInt(<string>process.env.CLEANUP_INTERVAL_SECONDS) || 1, 1) *
-  1000;
-
-export async function cleanExpiredNotes(): Promise<number> {
   logger.info("[Cleanup] Cleaning up expired notes...");
   const toDelete = await getExpiredNotes();
 
@@ -25,6 +22,8 @@ export async function cleanExpiredNotes(): Promise<number> {
         });
       });
       await Promise.all(logs);
+      const filter = await getExpiredNoteFilter();
+      await filter.addNoteIds(toDelete.map((n) => n.id));
       logger.info(`[Cleanup] Deleted ${deleteCount} expired notes.`);
       return deleteCount;
     })
@@ -34,3 +33,7 @@ export async function cleanExpiredNotes(): Promise<number> {
       return -1;
     });
 }
+
+export const deleteInterval =
+  Math.max(parseInt(<string>process.env.CLEANUP_INTERVAL_SECONDS) || 1, 1) *
+  1000;

server/src/tasks/deleteExpiredNotes.unit.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { deleteExpiredNotes } from "./deleteExpiredNotes";
+import * as noteDao from "../controllers/note/note.dao";
+import EventLogger from "../logging/EventLogger";
+import logger from "../logging/logger";
+import * as filter from "../lib/expiredNoteFilter";
+
+vi.mock("../controllers/note/note.dao", () => ({
+  getExpiredNotes: vi.fn(),
+  deleteNotes: vi.fn(),
+}));
+
+vi.mock("../lib/expiredNoteFilter", () => {
+  const instance = {
+    addNoteIds: vi.fn(),
+  };
+  return { getExpiredNoteFilter: () => instance };
+});
+
+vi.mock("../logging/EventLogger");
+
+vi.spyOn(logger, "error");
+
+describe("deleteExpiredNotes", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should call note dao", async () => {
+    // mock note dao
+    const mockedDao = vi.mocked(noteDao);
+    mockedDao.getExpiredNotes.mockResolvedValue([
+      {
+        id: "test",
+        ciphertext: "test",
+        hmac: "test",
+        insert_time: new Date(),
+        expire_time: new Date(),
+      },
+    ]);
+    mockedDao.deleteNotes.mockResolvedValue(1);
+
+    // mock ExpiredNoteFilter
+    const mockedFilter = vi.mocked(await filter.getExpiredNoteFilter());
+    mockedFilter.addNoteIds.mockResolvedValue();
+
+    // test task call
+    await deleteExpiredNotes();
+    expect(mockedDao.getExpiredNotes).toHaveBeenCalledOnce();
+    expect(mockedDao.deleteNotes).toHaveBeenCalledWith(["test"]);
+    expect(logger.error).not.toHaveBeenCalled();
+    expect(vi.mocked(EventLogger).purgeEvent).toHaveBeenCalledOnce();
+    expect(mockedFilter.addNoteIds).toHaveBeenCalledWith(["test"]);
+  });
+});

server/src/util.unit.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from "vitest";
-import { addDays } from "./util";
+import { addDays, getConnectingIp } from "./util";
 
 describe("addDays()", () => {
   it("Should add n days to the input date", () => {

webapp/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## [2022-08-11]
+
+- feat: ✨ Users receive a unique error page when opening expired notes vs. wrong URL. 
+
 ## [2022-08-07]
 
 - fix: 🐛 collapsed/uncollapsed callout syntax now correctly renders as a callout block instead of a block quote.

webapp/src/routes/__error.svelte

@@ -3,14 +3,22 @@
 
 	export const load: Load = ({ error, status }) => {
 		let explainText = '';
+		let title = '';
 
 		if (status == 404) {
-			explainText = `No note was found for this link. It may be that the note that was once connected to this link has expired.`;
+			title = `404: No note found 🕵️`;
+			explainText = `No note was found at this link. Are you from the future?`;
+		}
+
+		if (status == 410) {
+			title = '📝💨 This note is no longer here! ';
+			explainText = `Notes are stored for a limited amount of time. The note at this link was either set to expire, or deleted due to inactivity. Sorry!`;
 		}
 
 		return {
 			props: {
-				title: `${status}: ${error?.message}`,
+				status: status,
+				title: title,
 				explainText: explainText
 			}
 		};
@@ -18,6 +26,7 @@
 </script>
 
 <script lang="ts">
+	export let status: number;
 	export let title: string;
 	export let explainText: string;
 </script>
@@ -25,4 +34,10 @@
 <div class="prose max-w-2xl prose-zinc dark:prose-invert">
 	<h1>{title}</h1>
 	<p class="prose-xl">{explainText}</p>
+
+	<div class="not-prose w-full flex justify-center mt-16">
+		{#if status == 404 || status == 410}
+			<img src="/expired_note.svg" alt="encrypted-art" class="w-80" />
+		{/if}
+	</div>
 </div>

webapp/src/routes/note/[id].svelte

@@ -109,7 +109,7 @@
 
 {#if decryptFailed}
 	<div class="prose max-w-2xl prose-zinc dark:prose-invert">
-		<h1>Error: Cannot decrypt file</h1>
+		<h1>Error: Cannot decrypt file 🔒</h1>
 		<p class="prose-xl">This note could not be decrypted with this link.</p>
 		<p class="prose-xl">
 			If you think this is an error, please double check that you copied the entire URL.