Add storage cleaner

client/tsconfig.json

@@ -11,6 +11,7 @@
     "../server/log.ts",
     "../server/config.ts",
     "../server/client.ts",
+    "../server/storageCleaner.ts",
     "../server/clientManager.ts",
     "../server/identification.ts",
     "../server/plugins/changelog.ts",
@@ -44,14 +45,16 @@
   "compilerOptions": {
     "sourceMap": false /*Create source map files for emitted JavaScript files. See more: https://www.typescriptlang.org/tsconfig#sourceMap */,
     "jsx": "preserve" /* Specify what JSX code is generated. */,
-    "lib": ["DOM", "DOM.Iterable", "ESNext"],
-
+    "lib": [
+      "DOM",
+      "DOM.Iterable",
+      "ESNext"
+    ],
     // this enables stricter inference for data properties on `this`
     "strict": true,
     // if using webpack 2+ or rollup, to leverage tree shaking:
     "module": "es2020",
     "moduleResolution": "node",
-
     // TODO: Remove eventually
     "noImplicitAny": false /*Enable error reporting for expressions and declarations with an implied any type. See more: https://www.typescriptlang.org/tsconfig#noImplicitAny */
   } /* Instructs the TypeScript compiler how to compile .ts files. */

defaults/config.js

@@ -308,6 +308,26 @@ module.exports = {
 	// This value is set to `["sqlite", "text"]` by default.
 	messageStorage: ["sqlite", "text"],
 
+	// ### `storagePolicy`
+
+	// When the sqlite storage is in use, control the maximum storage duration.
+	// A background task will periodically clean up messages older than the limit.
+
+	// The available keys for the `storagePolicy` object are:
+	//
+	// - `enabled`: If this is false, the cleaning task is not running.
+	// - `maxAgeDays`: Maximum age of an entry in days.
+	// - `deletionPolicy`: Controls what types of messages are being deleted.
+	//   Valid options are:
+	//   - `statusOnly`: Only delete message types which are status related (e.g. away, back, join, parts, mode, ctcp...)
+	//     but keep actual messages from nicks. This keeps the DB size down while retaining "precious" messages.
+	//   - `everything`: Delete everything, including messages from irc nicks
+	storagePolicy: {
+		enabled: false,
+		maxAgeDays: 7,
+		deletionPolicy: "statusOnly",
+	},
+
 	// ### `useHexIp`
 	//
 	// When set to `true`, users' IP addresses will be encoded as hex.

server/config.ts

@@ -76,6 +76,12 @@ type Debug = {
 	raw: boolean;
 };
 
+type StoragePolicy = {
+	enabled: boolean;
+	maxAgeDays: number;
+	deletionPolicy: "statusOnly" | "everything";
+};
+
 export type ConfigType = {
 	public: boolean;
 	host: string | undefined;
@@ -97,6 +103,7 @@ export type ConfigType = {
 	defaults: Defaults;
 	lockNetwork: boolean;
 	messageStorage: string[];
+	storagePolicy: StoragePolicy;
 	useHexIp: boolean;
 	webirc?: WebIRC;
 	identd: Identd;

server/storageCleaner.ts

@@ -0,0 +1,148 @@
+import SqliteMessageStorage from "./plugins/messageStorage/sqlite";
+import {MessageType} from "./models/msg";
+import Config from "./config";
+import {DeletionRequest} from "./plugins/messageStorage/types";
+import log from "./log";
+
+const status_types = [
+	MessageType.AWAY,
+	MessageType.BACK,
+	MessageType.INVITE,
+	MessageType.JOIN,
+	MessageType.KICK,
+	MessageType.MODE,
+	MessageType.MODE_CHANNEL,
+	MessageType.MODE_USER,
+	MessageType.NICK,
+	MessageType.PART,
+	MessageType.QUIT,
+	MessageType.CTCP, // not technically a status, but generally those are only of interest temporarily
+	MessageType.CTCP_REQUEST,
+	MessageType.CHGHOST,
+	MessageType.TOPIC,
+	MessageType.TOPIC_SET_BY,
+];
+
+export class StorageCleaner {
+	db: SqliteMessageStorage;
+	olderThanDays: number;
+	messageTypes: MessageType[] | null;
+	limit: number;
+	ticker?: ReturnType<typeof setTimeout>;
+	errCount: number;
+	isStopped: boolean;
+
+	constructor(db: SqliteMessageStorage) {
+		this.errCount = 0;
+		this.isStopped = true;
+		this.db = db;
+		this.limit = 200;
+		const policy = Config.values.storagePolicy;
+		this.olderThanDays = policy.maxAgeDays;
+
+		switch (policy.deletionPolicy) {
+			case "statusOnly":
+				this.messageTypes = status_types;
+				break;
+			case "everything":
+				this.messageTypes = null;
+				break;
+			default:
+				// exhaustive switch guard, blows up when user specifies a invalid policy enum
+				this.messageTypes = assertNoBadPolicy(policy.deletionPolicy);
+		}
+	}
+
+	private genDeletionRequest(): DeletionRequest {
+		return {
+			limit: this.limit,
+			messageTypes: this.messageTypes,
+			olderThanDays: this.olderThanDays,
+		};
+	}
+
+	async runDeletesNoLimit(): Promise<number> {
+		if (!Config.values.storagePolicy.enabled) {
+			// this is meant to be used by cli tools, so we guard against this
+			throw new Error("storage policy is disabled");
+		}
+
+		const req = this.genDeletionRequest();
+		req.limit = -1; // unlimited
+		const num_deleted = await this.db.deleteMessages(req);
+		return num_deleted;
+	}
+
+	private async runDeletes() {
+		if (this.isStopped) {
+			return;
+		}
+
+		if (!this.db.isEnabled) {
+			// TODO: remove this once the server is intelligent enough to wait for init
+			this.schedule(30 * 1000);
+			return;
+		}
+
+		const req = this.genDeletionRequest();
+
+		let num_deleted = 0;
+
+		try {
+			num_deleted = await this.db.deleteMessages(req);
+			this.errCount = 0; // reset when it works
+		} catch (err: any) {
+			this.errCount++;
+			log.error("can't clean messages", err.message);
+
+			if (this.errCount === 2) {
+				log.error("Cleaning failed too many times, will not retry");
+				this.stop();
+				return;
+			}
+		}
+
+		// need to recheck here as the field may have changed since the await
+		if (this.isStopped) {
+			return;
+		}
+
+		if (num_deleted < req.limit) {
+			this.schedule(5 * 60 * 1000);
+		} else {
+			this.schedule(5000); // give others a chance to execute queries
+		}
+	}
+
+	private schedule(ms: number) {
+		const self = this;
+
+		this.ticker = setTimeout(() => {
+			self.runDeletes().catch((err) => {
+				log.error("storageCleaner: unexpected failure");
+				throw err;
+			});
+		}, ms);
+	}
+
+	start() {
+		this.isStopped = false;
+		this.schedule(0);
+	}
+
+	stop() {
+		this.isStopped = true;
+
+		if (!this.ticker) {
+			return;
+		}
+
+		clearTimeout(this.ticker);
+	}
+}
+
+function assertNoBadPolicy(_: never): never {
+	throw new Error(
+		`Invalid deletion policy "${Config.values.storagePolicy.deletionPolicy}" in the \`storagePolicy\` object, fix your config.`
+	);
+}