Add storage cleaner

This commit is contained in:
hgw 2024-02-02 03:05:47 +00:00
parent fb720d2080
commit b19562e53d
Signed by: hgw
SSH Key Fingerprint: SHA256:diG7RVYHjd3aDYkZWHYcBJbImu+6zfptuUP+3k/wol4
4 changed files with 182 additions and 4 deletions

View File

@ -11,6 +11,7 @@
"../server/log.ts",
"../server/config.ts",
"../server/client.ts",
"../server/storageCleaner.ts",
"../server/clientManager.ts",
"../server/identification.ts",
"../server/plugins/changelog.ts",
@ -44,14 +45,16 @@
"compilerOptions": {
"sourceMap": false /*Create source map files for emitted JavaScript files. See more: https://www.typescriptlang.org/tsconfig#sourceMap */,
"jsx": "preserve" /* Specify what JSX code is generated. */,
"lib": ["DOM", "DOM.Iterable", "ESNext"],
"lib": [
"DOM",
"DOM.Iterable",
"ESNext"
],
// this enables stricter inference for data properties on `this`
"strict": true,
// if using webpack 2+ or rollup, to leverage tree shaking:
"module": "es2020",
"moduleResolution": "node",
// TODO: Remove eventually
"noImplicitAny": false /*Enable error reporting for expressions and declarations with an implied any type. See more: https://www.typescriptlang.org/tsconfig#noImplicitAny */
} /* Instructs the TypeScript compiler how to compile .ts files. */

View File

@ -308,6 +308,26 @@ module.exports = {
// This value is set to `["sqlite", "text"]` by default.
messageStorage: ["sqlite", "text"],
// ### `storagePolicy`
// When the sqlite storage is in use, control the maximum storage duration.
// A background task will periodically clean up messages older than the limit.
// The available keys for the `storagePolicy` object are:
//
// - `enabled`: If this is false, the cleaning task is not running.
// - `maxAgeDays`: Maximum age of an entry in days.
// - `deletionPolicy`: Controls what types of messages are being deleted.
// Valid options are:
// - `statusOnly`: Only delete message types which are status related (e.g. away, back, join, parts, mode, ctcp...)
// but keep actual messages from nicks. This keeps the DB size down while retaining "precious" messages.
// - `everything`: Delete everything, including messages from irc nicks
storagePolicy: {
enabled: false,
maxAgeDays: 7,
deletionPolicy: "statusOnly",
},
// ### `useHexIp`
//
// When set to `true`, users' IP addresses will be encoded as hex.

View File

@ -76,6 +76,12 @@ type Debug = {
raw: boolean;
};
type StoragePolicy = {
enabled: boolean;
maxAgeDays: number;
deletionPolicy: "statusOnly" | "everything";
};
export type ConfigType = {
public: boolean;
host: string | undefined;
@ -97,6 +103,7 @@ export type ConfigType = {
defaults: Defaults;
lockNetwork: boolean;
messageStorage: string[];
storagePolicy: StoragePolicy;
useHexIp: boolean;
webirc?: WebIRC;
identd: Identd;

148
server/storageCleaner.ts Normal file
View File

@ -0,0 +1,148 @@
import SqliteMessageStorage from "./plugins/messageStorage/sqlite";
import {MessageType} from "./models/msg";
import Config from "./config";
import {DeletionRequest} from "./plugins/messageStorage/types";
import log from "./log";
const status_types = [
MessageType.AWAY,
MessageType.BACK,
MessageType.INVITE,
MessageType.JOIN,
MessageType.KICK,
MessageType.MODE,
MessageType.MODE_CHANNEL,
MessageType.MODE_USER,
MessageType.NICK,
MessageType.PART,
MessageType.QUIT,
MessageType.CTCP, // not technically a status, but generally those are only of interest temporarily
MessageType.CTCP_REQUEST,
MessageType.CHGHOST,
MessageType.TOPIC,
MessageType.TOPIC_SET_BY,
];
export class StorageCleaner {
db: SqliteMessageStorage;
olderThanDays: number;
messageTypes: MessageType[] | null;
limit: number;
ticker?: ReturnType<typeof setTimeout>;
errCount: number;
isStopped: boolean;
constructor(db: SqliteMessageStorage) {
this.errCount = 0;
this.isStopped = true;
this.db = db;
this.limit = 200;
const policy = Config.values.storagePolicy;
this.olderThanDays = policy.maxAgeDays;
switch (policy.deletionPolicy) {
case "statusOnly":
this.messageTypes = status_types;
break;
case "everything":
this.messageTypes = null;
break;
default:
// exhaustive switch guard, blows up when user specifies a invalid policy enum
this.messageTypes = assertNoBadPolicy(policy.deletionPolicy);
}
}
private genDeletionRequest(): DeletionRequest {
return {
limit: this.limit,
messageTypes: this.messageTypes,
olderThanDays: this.olderThanDays,
};
}
async runDeletesNoLimit(): Promise<number> {
if (!Config.values.storagePolicy.enabled) {
// this is meant to be used by cli tools, so we guard against this
throw new Error("storage policy is disabled");
}
const req = this.genDeletionRequest();
req.limit = -1; // unlimited
const num_deleted = await this.db.deleteMessages(req);
return num_deleted;
}
private async runDeletes() {
if (this.isStopped) {
return;
}
if (!this.db.isEnabled) {
// TODO: remove this once the server is intelligent enough to wait for init
this.schedule(30 * 1000);
return;
}
const req = this.genDeletionRequest();
let num_deleted = 0;
try {
num_deleted = await this.db.deleteMessages(req);
this.errCount = 0; // reset when it works
} catch (err: any) {
this.errCount++;
log.error("can't clean messages", err.message);
if (this.errCount === 2) {
log.error("Cleaning failed too many times, will not retry");
this.stop();
return;
}
}
// need to recheck here as the field may have changed since the await
if (this.isStopped) {
return;
}
if (num_deleted < req.limit) {
this.schedule(5 * 60 * 1000);
} else {
this.schedule(5000); // give others a chance to execute queries
}
}
private schedule(ms: number) {
const self = this;
this.ticker = setTimeout(() => {
self.runDeletes().catch((err) => {
log.error("storageCleaner: unexpected failure");
throw err;
});
}, ms);
}
start() {
this.isStopped = false;
this.schedule(0);
}
stop() {
this.isStopped = true;
if (!this.ticker) {
return;
}
clearTimeout(this.ticker);
}
}
function assertNoBadPolicy(_: never): never {
throw new Error(
`Invalid deletion policy "${Config.values.storagePolicy.deletionPolicy}" in the \`storagePolicy\` object, fix your config.`
);
}