From fb720d20803f7ed650b923bf1a86fe21df856c7e Mon Sep 17 00:00:00 2001 From: hgw Date: Fri, 2 Feb 2024 02:53:54 +0000 Subject: [PATCH] implement deleteMessages --- server/plugins/messageStorage/sqlite.ts | 37 +++++++- server/plugins/messageStorage/types.d.ts | 7 ++ test/plugins/sqlite.ts | 113 +++++++++++++++++++++++ 3 files changed, 155 insertions(+), 2 deletions(-) diff --git a/server/plugins/messageStorage/sqlite.ts b/server/plugins/messageStorage/sqlite.ts index b34c2417..5525d5bd 100644 --- a/server/plugins/messageStorage/sqlite.ts +++ b/server/plugins/messageStorage/sqlite.ts @@ -11,8 +11,10 @@ import type { SearchResponse, SearchQuery, SearchableMessageStorage, + DeletionRequest } from "./types"; import Network from "../../models/network"; +import { threadId } from "worker_threads"; // TODO; type let sqlite3: any; @@ -244,6 +246,10 @@ class SqliteMessageStorage implements SearchableMessageStorage { await this.serialize_run("VACUUM"); } + async vacuum() { + await this.serialize_run("VACUUM"); + } + async close() { if (!this.isEnabled) { return; @@ -499,6 +505,33 @@ class SqliteMessageStorage implements SearchableMessageStorage { }; } + async deleteMessages(req: DeletionRequest): Promise { + await this.initDone.promise; + let sql = "delete from messages where id in (select id from messages where\n"; + + // We roughly get a timestamp from N days before. + // We don't adjust for daylight savings time or other weird time jumps + const millisecondsInDay = 24 * 60 * 60 * 1000; + const deleteBefore = Date.now() - req.olderThanDays * millisecondsInDay; + sql += `time <= ${deleteBefore}\n`; + + let typeClause = ""; + + if (req.messageTypes !== null) { + typeClause = `type in (${req.messageTypes.map((type) => `'${type}'`).join(",")})\n`; + } + + if (typeClause) { + sql += `and ${typeClause}`; + } + + sql += "order by time asc\n"; + sql += `limit ${req.limit}\n`; + sql += ")"; + + return this.serialize_run(sql); + } + canProvideMessages() { return this.isEnabled; } @@ -506,13 +539,13 @@ class SqliteMessageStorage implements SearchableMessageStorage { private serialize_run(stmt: string, ...params: any[]): Promise { return new Promise((resolve, reject) => { this.database.serialize(() => { - this.database.run(stmt, params, (err) => { + this.database.run(stmt, params, function(err) { if (err) { reject(err); return; } - resolve(); + resolve(this.changes); // number of affected rows, 'this' is re-bound by sqlite3 }); }); }); diff --git a/server/plugins/messageStorage/types.d.ts b/server/plugins/messageStorage/types.d.ts index cc305224..2e037bea 100644 --- a/server/plugins/messageStorage/types.d.ts +++ b/server/plugins/messageStorage/types.d.ts @@ -4,6 +4,13 @@ import {Channel} from "../../models/channel"; import {Message} from "../../models/message"; import {Network} from "../../models/network"; import Client from "../../client"; +import type {MessageType} from "../../models/msg"; + +export type DeletionRequest = { + olderThanDays: number; + messageTypes: MessageType[] | null; //null means no restriction + limit: number; //-1 means unlimited +} interface MessageStorage { isEnabled: boolean; diff --git a/test/plugins/sqlite.ts b/test/plugins/sqlite.ts index 5f3ecc84..199e8b1e 100644 --- a/test/plugins/sqlite.ts +++ b/test/plugins/sqlite.ts @@ -12,6 +12,7 @@ import MessageStorage, { rollbacks, } from "../../server/plugins/messageStorage/sqlite"; import sqlite3 from "sqlite3"; +import {DeletionRequest} from "../../server/plugins/messageStorage/types" const orig_schema = [ // Schema version #1 @@ -127,6 +128,112 @@ describe("SQLite migrations", function () { }); }); +describe("SQLite unit tests", function () { + let store: MessageStorage; + + beforeEach(async function () { + store = new MessageStorage("testUser"); + await store._enable(":memory:"); + store.initDone.resolve(); + }); + + afterEach(async function () { + await store.close(); + }); + + it("deletes messages when asked to", async function () { + const baseDate = new Date(); + + const net = {uuid: "testnet"} as any; + const chan = {name: "#channel"} as any; + + for (let i = 0; i < 14; ++i) { + await store.index( + net, + chan, + new Msg({ + time: dateAddDays(baseDate, -i), + text: `msg ${i}`, + }) + ); + } + + const limit = 1; + const delReq: DeletionRequest = { + messageTypes: [MessageType.MESSAGE], + limit: limit, + olderThanDays: 2, + }; + + let deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(limit, "number of deleted messages doesn't match"); + + let id = 0; + let messages = await store.getMessages(net, chan, () => id++); + expect(messages.find((m) => m.text === "msg 13")).to.be.undefined; // oldest gets deleted first + + // let's test if it properly cleans now + delReq.limit = 100; + deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(11, "number of deleted messages doesn't match"); + messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.text)).to.have.ordered.members(["msg 1", "msg 0"]); + }); + + it("deletes only the types it should", async function () { + const baseDate = new Date(); + + const net = {uuid: "testnet"} as any; + const chan = {name: "#channel"} as any; + + for (let i = 0; i < 6; ++i) { + await store.index( + net, + chan, + new Msg({ + time: dateAddDays(baseDate, -i), + text: `msg ${i}`, + type: [ + MessageType.ACTION, + MessageType.AWAY, + MessageType.JOIN, + MessageType.PART, + MessageType.KICK, + MessageType.MESSAGE, + ][i], + }) + ); + } + + const delReq: DeletionRequest = { + messageTypes: [MessageType.ACTION, MessageType.JOIN, MessageType.KICK], + limit: 100, // effectively no limit + olderThanDays: 0, + }; + + let deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(3, "number of deleted messages doesn't match"); + + let id = 0; + let messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.type)).to.have.ordered.members([ + MessageType.MESSAGE, + MessageType.PART, + MessageType.AWAY, + ]); + + delReq.messageTypes = [ + MessageType.JOIN, // this is not in the remaining set, just here as a dummy + MessageType.PART, + MessageType.MESSAGE, + ]; + deleted = await store.deleteMessages(delReq); + expect(deleted).to.equal(2, "number of deleted messages doesn't match"); + messages = await store.getMessages(net, chan, () => id++); + expect(messages.map((m) => m.type)).to.have.ordered.members([MessageType.AWAY]); + }); +}); + describe("SQLite Message Storage", function () { // Increase timeout due to unpredictable I/O on CI services this.timeout(util.isRunningOnCI() ? 25000 : 5000); @@ -388,3 +495,9 @@ describe("SQLite Message Storage", function () { expect(fs.existsSync(expectedPath)).to.be.true; }); }); + +function dateAddDays(date: Date, days: number) { + const ret = new Date(date.valueOf()); + ret.setDate(date.getDate() + days); + return ret; +}