From 908c6b590156cc5840b544ea9d5374711d28729d Mon Sep 17 00:00:00 2001 From: Maxime Cannoodt Date: Tue, 9 Aug 2022 22:42:03 +0200 Subject: [PATCH] feat: :sparkles: return HTTP 410 page for expired notes --- server/package-lock.json | 162 ++++++++++++++++++ server/package.json | 3 +- .../migration.sql | 5 + server/prisma/schema.prisma | 5 + server/src/app.integration.test.ts | 10 +- server/src/app.ts | 4 +- .../controllers/note/note.get.controller.ts | 28 ++- .../db/bloomFilter.dao.integration.test.ts | 45 +++++ server/src/db/bloomFilter.dao.ts | 68 ++++++++ server/src/db/client.ts | 4 + server/src/lib/expiredNoteFilter.ts | 58 +++++++ server/src/lib/expiredNoteFilter.unit.test.ts | 93 ++++++++++ server/src/logging/__mocks__/EventLogger.ts | 9 + server/src/tasks/deleteExpiredNotes.ts | 13 +- .../src/tasks/deleteExpiredNotes.unit.test.ts | 55 ++++++ server/src/util.unit.test.ts | 2 +- webapp/CHANGELOG.md | 4 + webapp/src/routes/__error.svelte | 19 +- webapp/src/routes/note/[id].svelte | 2 +- webapp/static/expired_note.svg | 1 + 20 files changed, 566 insertions(+), 24 deletions(-) create mode 100644 server/prisma/migrations/20220809194448_add_bloom_filter_table/migration.sql create mode 100644 server/src/db/bloomFilter.dao.integration.test.ts create mode 100644 server/src/db/bloomFilter.dao.ts create mode 100644 server/src/lib/expiredNoteFilter.ts create mode 100644 server/src/lib/expiredNoteFilter.unit.test.ts create mode 100644 server/src/logging/__mocks__/EventLogger.ts create mode 100644 server/src/tasks/deleteExpiredNotes.unit.test.ts create mode 100644 webapp/static/expired_note.svg diff --git a/server/package-lock.json b/server/package-lock.json index 13c9b68..3e913d9 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -10,6 +10,7 @@ "license": "MIT", "dependencies": { "@prisma/client": "^4.0.0", + "bloom-filters": "^3.0.0", "body-parser": "^1.20.0", "class-validator": "^0.13.2", "dotenv": "^16.0.1", @@ -640,6 +641,14 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-arraybuffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", + "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -649,6 +658,25 @@ "node": ">=8" } }, + "node_modules/bloom-filters": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bloom-filters/-/bloom-filters-3.0.0.tgz", + "integrity": "sha512-DBDgLkYokKS5NA5y8P9fuTavKQCkleAP39yqpW/5Nab/vwzHv+wOPRM/yDAStghARDleyRI4orW91uuxj48LKQ==", + "dependencies": { + "base64-arraybuffer": "^1.0.2", + "is-buffer": "^2.0.5", + "lodash": "^4.17.15", + "lodash.eq": "^4.0.0", + "lodash.indexof": "^4.0.5", + "long": "^5.2.0", + "reflect-metadata": "^0.1.13", + "seedrandom": "^3.0.5", + "xxhashjs": "^0.2.2" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -1018,6 +1046,11 @@ "node": ">= 8" } }, + "node_modules/cuint": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==" + }, "node_modules/debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -2357,6 +2390,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "engines": { + "node": ">=4" + } + }, "node_modules/is-callable": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", @@ -2644,6 +2699,26 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/lodash.eq": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.eq/-/lodash.eq-4.0.0.tgz", + "integrity": "sha512-vbrJpXL6kQNG6TkInxX12DZRfuYVllSxhwYqjYB78g2zF3UI15nFO/0AgmZnZRnaQ38sZtjCiVjGr2rnKt4v0g==" + }, + "node_modules/lodash.indexof": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/lodash.indexof/-/lodash.indexof-4.0.5.tgz", + "integrity": "sha512-t9wLWMQsawdVmf6/IcAgVGqAJkNzYVcn4BHYZKTPW//l7N5Oq7Bq138BaVk19agcsPZePcidSgTTw4NqS1nUAw==" + }, + "node_modules/long": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", + "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" + }, "node_modules/loupe": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz", @@ -3841,6 +3916,11 @@ "tslib": "^1.9.3" } }, + "node_modules/reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" + }, "node_modules/regexp.prototype.flags": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", @@ -3954,6 +4034,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "node_modules/semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -5039,6 +5124,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/xxhashjs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", + "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", + "dependencies": { + "cuint": "^0.2.2" + } + }, "node_modules/y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", @@ -5598,12 +5691,33 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-arraybuffer": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz", + "integrity": "sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ==" + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "bloom-filters": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/bloom-filters/-/bloom-filters-3.0.0.tgz", + "integrity": "sha512-DBDgLkYokKS5NA5y8P9fuTavKQCkleAP39yqpW/5Nab/vwzHv+wOPRM/yDAStghARDleyRI4orW91uuxj48LKQ==", + "requires": { + "base64-arraybuffer": "^1.0.2", + "is-buffer": "^2.0.5", + "lodash": "^4.17.15", + "lodash.eq": "^4.0.0", + "lodash.indexof": "^4.0.5", + "long": "^5.2.0", + "reflect-metadata": "^0.1.13", + "seedrandom": "^3.0.5", + "xxhashjs": "^0.2.2" + } + }, "body-parser": { "version": "1.20.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.0.tgz", @@ -5905,6 +6019,11 @@ "which": "^2.0.1" } }, + "cuint": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/cuint/-/cuint-0.2.2.tgz", + "integrity": "sha512-d4ZVpCW31eWwCMe1YT3ur7mUDnTXbgwyzaL320DrcRT45rfjYxkt5QWLrmOJ+/UEAI2+fQgKe/fCjR8l4TpRgw==" + }, "debug": { "version": "2.6.9", "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", @@ -6826,6 +6945,11 @@ "has-tostringtag": "^1.0.0" } }, + "is-buffer": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-2.0.5.tgz", + "integrity": "sha512-i2R6zNFDwgEHJyQUtJEk0XFi1i0dPFn/oqjK3/vPCcDeJvW5NQ83V8QbicfF1SupOaB0h8ntgBC2YiE7dfyctQ==" + }, "is-callable": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.4.tgz", @@ -7020,6 +7144,26 @@ "p-locate": "^5.0.0" } }, + "lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "lodash.eq": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/lodash.eq/-/lodash.eq-4.0.0.tgz", + "integrity": "sha512-vbrJpXL6kQNG6TkInxX12DZRfuYVllSxhwYqjYB78g2zF3UI15nFO/0AgmZnZRnaQ38sZtjCiVjGr2rnKt4v0g==" + }, + "lodash.indexof": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/lodash.indexof/-/lodash.indexof-4.0.5.tgz", + "integrity": "sha512-t9wLWMQsawdVmf6/IcAgVGqAJkNzYVcn4BHYZKTPW//l7N5Oq7Bq138BaVk19agcsPZePcidSgTTw4NqS1nUAw==" + }, + "long": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.0.tgz", + "integrity": "sha512-9RTUNjK60eJbx3uz+TEGF7fUr29ZDxR5QzXcyDpeSfeH28S9ycINflOgOlppit5U+4kNTe83KQnMEerw7GmE8w==" + }, "loupe": { "version": "2.3.4", "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.4.tgz", @@ -7907,6 +8051,11 @@ "tslib": "^1.9.3" } }, + "reflect-metadata": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/reflect-metadata/-/reflect-metadata-0.1.13.tgz", + "integrity": "sha512-Ts1Y/anZELhSsjMcU605fU9RE4Oi3p5ORujwbIKXfWa+0Zxs510Qrmrce5/Jowq3cHSZSJqBjypxmHarc+vEWg==" + }, "regexp.prototype.flags": { "version": "1.4.3", "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.4.3.tgz", @@ -7973,6 +8122,11 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "semver": { "version": "6.3.0", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", @@ -8761,6 +8915,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "xxhashjs": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/xxhashjs/-/xxhashjs-0.2.2.tgz", + "integrity": "sha512-AkTuIuVTET12tpsVIQo+ZU6f/qDmKuRUcjaqR+OIvm+aCBsZ95i7UVY5WJ9TMsSaZ0DA2WxoZ4acu0sPH+OKAw==", + "requires": { + "cuint": "^0.2.2" + } + }, "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", diff --git a/server/package.json b/server/package.json index fdeeae9..49af4cf 100644 --- a/server/package.json +++ b/server/package.json @@ -6,7 +6,7 @@ "scripts": { "test": "run-s test:db:reset test:test", "coverage": "run-s test:db:reset test:coverage", - "test-watch": "dotenv -e .env.test -- vitest unit --coverage", + "test-watch": "dotenv -v UNIT_TEST=TRUE -e .env.test -- vitest unit", "test:test": "dotenv -e .env.test -- vitest run --no-threads", "test:coverage": "dotenv -e .env.test -- vitest run --no-threads --coverage", "test:db:reset": "dotenv -e .env.test -- npx prisma migrate reset -f", @@ -17,6 +17,7 @@ "license": "MIT", "dependencies": { "@prisma/client": "^4.0.0", + "bloom-filters": "^3.0.0", "body-parser": "^1.20.0", "class-validator": "^0.13.2", "dotenv": "^16.0.1", diff --git a/server/prisma/migrations/20220809194448_add_bloom_filter_table/migration.sql b/server/prisma/migrations/20220809194448_add_bloom_filter_table/migration.sql new file mode 100644 index 0000000..2ab393f --- /dev/null +++ b/server/prisma/migrations/20220809194448_add_bloom_filter_table/migration.sql @@ -0,0 +1,5 @@ +-- CreateTable +CREATE TABLE "BloomFilter" ( + "name" TEXT NOT NULL PRIMARY KEY, + "serializedFilter" BLOB NOT NULL +); diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index e4a40ef..1f0b268 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -29,3 +29,8 @@ model event { error String? expire_window_days Int? } + +model BloomFilter { + name String @id + serializedFilter Bytes +} diff --git a/server/src/app.integration.test.ts b/server/src/app.integration.test.ts index 78a701e..7b11163 100644 --- a/server/src/app.integration.test.ts +++ b/server/src/app.integration.test.ts @@ -2,8 +2,10 @@ import { app } from "./app"; import supertest from "supertest"; import { describe, it, expect } from "vitest"; import prisma from "./db/client"; -import { cleanExpiredNotes } from "./tasks/deleteExpiredNotes"; +import { deleteExpiredNotes } from "./tasks/deleteExpiredNotes"; import { EventType } from "./logging/EventLogger"; +import { getFilter } from "./db/bloomFilter.dao"; +import { getExpiredNoteFilter } from "./lib/expiredNoteFilter"; // const testNote with base64 ciphertext and hmac const testNote = { @@ -182,12 +184,12 @@ describe("Clean expired notes", () => { expect(res.statusCode).toBe(200); // run cleanup - const nDeleted = await cleanExpiredNotes(); + const nDeleted = await deleteExpiredNotes(); expect(nDeleted).toBeGreaterThan(0); - // make sure note is gone + // if the note is added to the expire filter, it returns 410 res = await supertest(app).get(`/api/note/${id}`); - expect(res.statusCode).toBe(404); + expect(res.statusCode).toBe(410); // sleep 100ms to allow all events to be logged await new Promise((resolve) => setTimeout(resolve, 200)); diff --git a/server/src/app.ts b/server/src/app.ts index 5394e84..56a8612 100644 --- a/server/src/app.ts +++ b/server/src/app.ts @@ -4,7 +4,7 @@ import helmet from "helmet"; import pinoHttp from "pino-http"; import logger from "./logging/logger"; import { notesRoute } from "./controllers/note/note.router"; -import { cleanExpiredNotes, cleanInterval } from "./tasks/deleteExpiredNotes"; +import { deleteExpiredNotes, deleteInterval } from "./tasks/deleteExpiredNotes"; // Initialize middleware clients export const app: Express = express(); @@ -32,4 +32,4 @@ app.use( app.use("/api/note/", notesRoute); // Run periodic tasks -setInterval(cleanExpiredNotes, cleanInterval); +setInterval(deleteExpiredNotes, deleteInterval); diff --git a/server/src/controllers/note/note.get.controller.ts b/server/src/controllers/note/note.get.controller.ts index fa4ecc9..9ef3b47 100644 --- a/server/src/controllers/note/note.get.controller.ts +++ b/server/src/controllers/note/note.get.controller.ts @@ -1,8 +1,8 @@ import { NextFunction, Request, Response } from "express"; +import { getExpiredNoteFilter } from "../../lib/expiredNoteFilter"; import EventLogger from "../../logging/EventLogger"; import { getConnectingIp } from "../../util"; import { getNote } from "./note.dao"; - export async function getNoteController( req: Request, res: Response, @@ -20,13 +20,25 @@ export async function getNoteController( }); res.send(note); } else { - await EventLogger.readEvent({ - success: false, - host: ip, - note_id: req.params.id, - error: "Note not found", - }); - res.status(404).send(); + // check the expired filter to see if the note was expired + const expiredFilter = await getExpiredNoteFilter(); + if (expiredFilter.hasNoteId(req.params.id)) { + await EventLogger.readEvent({ + success: false, + host: ip, + note_id: req.params.id, + error: "Note expired", + }); + res.status(410).send("Note expired"); + } else { + await EventLogger.readEvent({ + success: false, + host: ip, + note_id: req.params.id, + error: "Note not found", + }); + res.status(404).send(); + } } }) .catch(async (err) => { diff --git a/server/src/db/bloomFilter.dao.integration.test.ts b/server/src/db/bloomFilter.dao.integration.test.ts new file mode 100644 index 0000000..8a937fc --- /dev/null +++ b/server/src/db/bloomFilter.dao.integration.test.ts @@ -0,0 +1,45 @@ +import { ScalableBloomFilter } from "bloom-filters"; +import { describe, it, expect } from "vitest"; +import { getFilter, upsertFilter } from "./bloomFilter.dao"; + +describe("Serialization and deserialization of filters", () => { + it("Should serialize and deserialize filters", async () => { + // Serialize a new filter + const testFilter = new ScalableBloomFilter(); + testFilter.add("item0"); + testFilter.add("item1"); + testFilter.add("item2"); + await upsertFilter("testFilter", testFilter); + + // Deserialize the filter + const deserializedFilter = await getFilter( + "testFilter", + ScalableBloomFilter + ); + expect(deserializedFilter.has("item0")).toBe(true); + expect(deserializedFilter.has("item1")).toBe(true); + expect(deserializedFilter.has("item2")).toBe(true); + expect(deserializedFilter.has("not_in_filter")).toBe(false); + + // update the filter and serialize it + deserializedFilter.add("item3"); + await upsertFilter("testFilter", deserializedFilter); + + // Deserialize the filter again + const deserializedFilter2 = await getFilter( + "testFilter", + ScalableBloomFilter + ); + expect(deserializedFilter2.has("item0")).toBe(true); + expect(deserializedFilter2.has("item1")).toBe(true); + expect(deserializedFilter2.has("item2")).toBe(true); + expect(deserializedFilter2.has("item3")).toBe(true); + expect(deserializedFilter2.has("not_in_filter")).toBe(false); + }); + + it("Should throw error if no filter is saved for given name", async () => { + await expect( + getFilter("not_existing", ScalableBloomFilter) + ).rejects.toThrow("No BloomFilter found"); + }); +}); diff --git a/server/src/db/bloomFilter.dao.ts b/server/src/db/bloomFilter.dao.ts new file mode 100644 index 0000000..5437269 --- /dev/null +++ b/server/src/db/bloomFilter.dao.ts @@ -0,0 +1,68 @@ +import { BaseFilter } from "bloom-filters"; +import prisma from "../db/client"; + +interface IDeserializedFilter { + fromJSON: (json: JSON) => BaseFilter; +} + +/** + * Get a bloom filter from the database. + * @param name Name of the filter in database + * @param cls Class of the filter to deserialize + * @returns Deserialized filter of type T + * @throws Error if no filter is found in database + */ +export async function getFilter( + name: string, + cls: IDeserializedFilter +): Promise { + const bloomFilter = await prisma.bloomFilter.findUniqueOrThrow({ + where: { + name: name, + }, + }); + const serializedFilter = bloomFilter.serializedFilter; + return deserializeFilter(serializedFilter, cls); +} + +/** + * Creates a filter in the database if it does not exist yet. + * If it exists, it will be overwritten. + * @param name Name of the filter in database + * @param filter filter object to serialize + */ +export async function upsertFilter( + name: string, + filter: BaseFilter +): Promise { + const serializedFilter = serializeFilter(filter); + await prisma.bloomFilter.upsert({ + where: { + name: name, + }, + update: { + serializedFilter: serializedFilter, + }, + create: { + name: name, + serializedFilter: serializedFilter, + }, + }); +} + +function serializeFilter(filter: BaseFilter): Buffer { + const filterJSON = filter.saveAsJSON(); + const filterString = JSON.stringify(filterJSON); + const filterBuffer = Buffer.from(filterString, "utf-8"); + return filterBuffer; +} + +function deserializeFilter( + serializedFilter: Buffer, + cls: IDeserializedFilter +): T { + const filterString = serializedFilter.toString("utf-8"); + const filterJSON = JSON.parse(filterString); + const filter = cls.fromJSON(filterJSON) as T; + return filter; +} diff --git a/server/src/db/client.ts b/server/src/db/client.ts index b5bf6ce..41cfdc3 100644 --- a/server/src/db/client.ts +++ b/server/src/db/client.ts @@ -1,5 +1,9 @@ import { PrismaClient } from "@prisma/client"; +if (process.env.UNIT_TEST === "TRUE") { + throw Error("Database operations must be mocked in unit tests."); +} + const prisma = new PrismaClient(); export default prisma; diff --git a/server/src/lib/expiredNoteFilter.ts b/server/src/lib/expiredNoteFilter.ts new file mode 100644 index 0000000..60c8466 --- /dev/null +++ b/server/src/lib/expiredNoteFilter.ts @@ -0,0 +1,58 @@ +import { ScalableBloomFilter } from "bloom-filters"; +import { getFilter, upsertFilter } from "../db/bloomFilter.dao"; + +export class ExpiredNoteFilter { + _filter: ScalableBloomFilter; + static FILTER_NAME = "expiredNotes"; + + private constructor(filter: ScalableBloomFilter) { + this._filter = filter; + } + + public static async deserializeFromDb(): Promise { + return ExpiredNoteFilter._deserializeFilter() + .catch((err) => { + if (err.message === "No BloomFilter found") { + return new ScalableBloomFilter(); + } else { + throw err; + } + }) + .then((filter) => { + return new ExpiredNoteFilter(filter); + }); + } + + public addNoteIds(noteIds: string[]): Promise { + noteIds.forEach((noteId) => { + this._filter.add(noteId); + }); + return this._serialize(); + } + + public hasNoteId(noteId: string): boolean { + return this._filter.has(noteId); + } + + private _serialize(): Promise { + return upsertFilter(ExpiredNoteFilter.FILTER_NAME, this._filter); + } + + private static _deserializeFilter(): Promise { + return getFilter( + this.FILTER_NAME, + ScalableBloomFilter + ); + } +} + +let _filter: ExpiredNoteFilter; + +export async function getExpiredNoteFilter(): Promise { + if (_filter) { + return _filter; + } else { + _filter = await ExpiredNoteFilter.deserializeFromDb(); + return _filter; + } +} diff --git a/server/src/lib/expiredNoteFilter.unit.test.ts b/server/src/lib/expiredNoteFilter.unit.test.ts new file mode 100644 index 0000000..c2bd405 --- /dev/null +++ b/server/src/lib/expiredNoteFilter.unit.test.ts @@ -0,0 +1,93 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { ExpiredNoteFilter } from "./expiredNoteFilter"; +import { ScalableBloomFilter } from "bloom-filters"; + +import * as dao from "../db/bloomFilter.dao"; +vi.mock("../db/bloomFilter.dao", () => ({ + getFilter: vi.fn(), + upsertFilter: vi.fn(), +})); + +describe("Deserialization from database", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should create a new filter if no filter is found in database", async () => { + // mock no filter in database + const mockedDao = vi.mocked(dao); + mockedDao.getFilter.mockRejectedValue(new Error("No BloomFilter found")); + + // test instatiation + const testFilter = await ExpiredNoteFilter.deserializeFromDb(); + expect(mockedDao.getFilter).toHaveBeenCalledWith( + "expiredNotes", + ScalableBloomFilter + ); + expect(testFilter).toBeInstanceOf(ExpiredNoteFilter); + + // expect the _filter property to be a fresh ScalableBloomFilter (capacity 8) + expect(testFilter._filter).toBeInstanceOf(ScalableBloomFilter); + expect(testFilter._filter.capacity()).toBe(8); + }); + + it("should deserialize a filter from the database", async () => { + // mock prefilled bloom filter + const bloomFilter = new ScalableBloomFilter(); + bloomFilter.add("test"); + bloomFilter.add("test2"); + const mockedDao = vi.mocked(dao); + mockedDao.getFilter.mockResolvedValue(bloomFilter); + + // test instatiation + const testFilter = await ExpiredNoteFilter.deserializeFromDb(); + expect(mockedDao.getFilter).toHaveBeenCalledWith( + "expiredNotes", + ScalableBloomFilter + ); + expect(testFilter._filter).toBe(bloomFilter); + + // expect the testFilter to have the same content as the mocked bloom filter + expect(testFilter.hasNoteId("test")).toBe(true); + expect(testFilter.hasNoteId("test2")).toBe(true); + }); +}); + +describe("Filter operations and serialization", () => { + let testFilter: ExpiredNoteFilter; + + beforeEach(async () => { + const mockedDao = vi.mocked(dao); + mockedDao.getFilter.mockResolvedValue(new ScalableBloomFilter()); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should add multiple noteIds to the filter", async () => { + testFilter = await ExpiredNoteFilter.deserializeFromDb(); + testFilter.addNoteIds(["test", "test2"]); + expect(testFilter.hasNoteId("test")).toBe(true); + expect(testFilter.hasNoteId("test2")).toBe(true); + + // expect the filter to be serialized to database + const mockedDao = vi.mocked(dao); + expect(mockedDao.upsertFilter).toHaveBeenCalledWith( + "expiredNotes", + testFilter._filter + ); + }); + + it("Should have an error rate <1% for 1000 elements", async () => { + testFilter = await ExpiredNoteFilter.deserializeFromDb(); + const elements = Array.from({ length: 1000 }, (_, i) => i.toString()); + testFilter.addNoteIds(elements); + + // expect the filter to have an error rate <1% + const errorRate = elements + .map((el) => (testFilter.hasNoteId(el) ? 0 : 1)) + .reduce((acc, cur) => acc + cur, 0); + expect(errorRate).toBeLessThan(10); + }); +}); diff --git a/server/src/logging/__mocks__/EventLogger.ts b/server/src/logging/__mocks__/EventLogger.ts new file mode 100644 index 0000000..3c2e2b3 --- /dev/null +++ b/server/src/logging/__mocks__/EventLogger.ts @@ -0,0 +1,9 @@ +import { vi } from "vitest"; + +const mockedEventLogger = { + writeEvent: vi.fn(), + readEvent: vi.fn(), + purgeEvent: vi.fn(), +}; + +export default mockedEventLogger; diff --git a/server/src/tasks/deleteExpiredNotes.ts b/server/src/tasks/deleteExpiredNotes.ts index 64a7a71..9d37850 100644 --- a/server/src/tasks/deleteExpiredNotes.ts +++ b/server/src/tasks/deleteExpiredNotes.ts @@ -1,12 +1,9 @@ import { deleteNotes, getExpiredNotes } from "../controllers/note/note.dao"; +import { getExpiredNoteFilter } from "../lib/expiredNoteFilter"; import EventLogger from "../logging/EventLogger"; import logger from "../logging/logger"; -export const cleanInterval = - Math.max(parseInt(process.env.CLEANUP_INTERVAL_SECONDS) || 1, 1) * - 1000; - -export async function cleanExpiredNotes(): Promise { +export async function deleteExpiredNotes(): Promise { logger.info("[Cleanup] Cleaning up expired notes..."); const toDelete = await getExpiredNotes(); @@ -25,6 +22,8 @@ export async function cleanExpiredNotes(): Promise { }); }); await Promise.all(logs); + const filter = await getExpiredNoteFilter(); + await filter.addNoteIds(toDelete.map((n) => n.id)); logger.info(`[Cleanup] Deleted ${deleteCount} expired notes.`); return deleteCount; }) @@ -34,3 +33,7 @@ export async function cleanExpiredNotes(): Promise { return -1; }); } + +export const deleteInterval = + Math.max(parseInt(process.env.CLEANUP_INTERVAL_SECONDS) || 1, 1) * + 1000; diff --git a/server/src/tasks/deleteExpiredNotes.unit.test.ts b/server/src/tasks/deleteExpiredNotes.unit.test.ts new file mode 100644 index 0000000..ed2f7c9 --- /dev/null +++ b/server/src/tasks/deleteExpiredNotes.unit.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { deleteExpiredNotes } from "./deleteExpiredNotes"; +import * as noteDao from "../controllers/note/note.dao"; +import EventLogger from "../logging/EventLogger"; +import logger from "../logging/logger"; +import * as filter from "../lib/expiredNoteFilter"; + +vi.mock("../controllers/note/note.dao", () => ({ + getExpiredNotes: vi.fn(), + deleteNotes: vi.fn(), +})); + +vi.mock("../lib/expiredNoteFilter", () => { + const instance = { + addNoteIds: vi.fn(), + }; + return { getExpiredNoteFilter: () => instance }; +}); + +vi.mock("../logging/EventLogger"); + +vi.spyOn(logger, "error"); + +describe("deleteExpiredNotes", () => { + afterEach(() => { + vi.clearAllMocks(); + }); + + it("should call note dao", async () => { + // mock note dao + const mockedDao = vi.mocked(noteDao); + mockedDao.getExpiredNotes.mockResolvedValue([ + { + id: "test", + ciphertext: "test", + hmac: "test", + insert_time: new Date(), + expire_time: new Date(), + }, + ]); + mockedDao.deleteNotes.mockResolvedValue(1); + + // mock ExpiredNoteFilter + const mockedFilter = vi.mocked(await filter.getExpiredNoteFilter()); + mockedFilter.addNoteIds.mockResolvedValue(); + + // test task call + await deleteExpiredNotes(); + expect(mockedDao.getExpiredNotes).toHaveBeenCalledOnce(); + expect(mockedDao.deleteNotes).toHaveBeenCalledWith(["test"]); + expect(logger.error).not.toHaveBeenCalled(); + expect(vi.mocked(EventLogger).purgeEvent).toHaveBeenCalledOnce(); + expect(mockedFilter.addNoteIds).toHaveBeenCalledWith(["test"]); + }); +}); diff --git a/server/src/util.unit.test.ts b/server/src/util.unit.test.ts index d78d8bf..f9bde62 100644 --- a/server/src/util.unit.test.ts +++ b/server/src/util.unit.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from "vitest"; -import { addDays } from "./util"; +import { addDays, getConnectingIp } from "./util"; describe("addDays()", () => { it("Should add n days to the input date", () => { diff --git a/webapp/CHANGELOG.md b/webapp/CHANGELOG.md index 9e62e66..5f9fad8 100644 --- a/webapp/CHANGELOG.md +++ b/webapp/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [2022-08-11] + +- feat: ✨ Users receive a unique error page when opening expired notes vs. wrong URL. + ## [2022-08-07] - fix: 🐛 collapsed/uncollapsed callout syntax now correctly renders as a callout block instead of a block quote. diff --git a/webapp/src/routes/__error.svelte b/webapp/src/routes/__error.svelte index f729963..c16160d 100644 --- a/webapp/src/routes/__error.svelte +++ b/webapp/src/routes/__error.svelte @@ -3,14 +3,22 @@ export const load: Load = ({ error, status }) => { let explainText = ''; + let title = ''; if (status == 404) { - explainText = `No note was found for this link. It may be that the note that was once connected to this link has expired.`; + title = `404: No note found 🕵️`; + explainText = `No note was found at this link. Are you from the future?`; + } + + if (status == 410) { + title = '📝💨 This note is no longer here! '; + explainText = `Notes are stored for a limited amount of time. The note at this link was either set to expire, or deleted due to inactivity. Sorry!`; } return { props: { - title: `${status}: ${error?.message}`, + status: status, + title: title, explainText: explainText } }; @@ -18,6 +26,7 @@ @@ -25,4 +34,10 @@

{title}

{explainText}

+ +
+ {#if status == 404 || status == 410} + encrypted-art + {/if} +
diff --git a/webapp/src/routes/note/[id].svelte b/webapp/src/routes/note/[id].svelte index d905991..25dd913 100644 --- a/webapp/src/routes/note/[id].svelte +++ b/webapp/src/routes/note/[id].svelte @@ -109,7 +109,7 @@ {#if decryptFailed}
-

Error: Cannot decrypt file

+

Error: Cannot decrypt file 🔒

This note could not be decrypted with this link.

If you think this is an error, please double check that you copied the entire URL. diff --git a/webapp/static/expired_note.svg b/webapp/static/expired_note.svg new file mode 100644 index 0000000..8174fed --- /dev/null +++ b/webapp/static/expired_note.svg @@ -0,0 +1 @@ + \ No newline at end of file