From 72fabd5346e23621e52b5acbac5fe4375f981d16 Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:02:20 +0000 Subject: [PATCH 1/7] Upgrade Dependencies --- package.json | 14 ++++++------ yarn.lock | 61 ++++++++++++++++++++++++++++------------------------ 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index 59f95046..a05b558f 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "dependencies": { "@fastify/busboy": "1.0.0", "bcryptjs": "2.4.3", + "caniuse-lite": "^1.0.30001561", "chalk": "4.1.2", "cheerio": "1.0.0-rc.12", "commander": "9.0.0", @@ -61,6 +62,7 @@ "express": "4.17.3", "file-type": "16.5.4", "filenamify": "4.3.0", + "get-func-name": "2.0.2", "got": "11.8.5", "irc-framework": "4.13.1", "is-utf8": "0.2.1", @@ -91,18 +93,18 @@ "@istanbuljs/nyc-config-typescript": "1.0.2", "@textcomplete/core": "0.1.10", "@textcomplete/textarea": "0.1.12", - "@types/bcryptjs": "2.4.4", + "@types/bcryptjs": "2.4.5", "@types/chai": "4.3.5", - "@types/cheerio": "0.22.31", - "@types/content-disposition": "0.5.5", + "@types/cheerio": "0.22.33", + "@types/content-disposition": "0.5.7", "@types/express": "4.17.13", - "@types/is-utf8": "0.2.1", + "@types/is-utf8": "0.2.2", "@types/ldapjs": "2.2.2", "@types/linkify-it": "3.0.3", - "@types/lodash": "4.14.195", + "@types/lodash": "4.14.200", "@types/mime-types": "2.1.1", "@types/mocha": "9.1.1", - "@types/mousetrap": "1.6.11", + "@types/mousetrap": "1.6.13", "@types/node": "17.0.31", "@types/read": "0.0.29", "@types/semver": "7.3.9", diff --git a/yarn.lock b/yarn.lock index 1be94e64..354e8ab4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1328,10 +1328,10 @@ resolved "https://registry.yarnpkg.com/@tsconfig/node16/-/node16-1.0.4.tgz#0b92dcc0cc1c81f6f306a381f28e31b1a56536e9" integrity sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA== -"@types/bcryptjs@2.4.4": - version "2.4.4" - resolved "https://registry.yarnpkg.com/@types/bcryptjs/-/bcryptjs-2.4.4.tgz#cd3c4c007f600f1d21db09c9bd4ced8b49d04670" - integrity sha512-9wlJI7k5gRyJEC4yrV7DubzNQFTPiykYxUA6lBtsk5NlOfW9oWLJ1HdIA4YtE+6C3i3mTpDQQEosJ2rVZfBWnw== +"@types/bcryptjs@2.4.5": + version "2.4.5" + resolved "https://registry.yarnpkg.com/@types/bcryptjs/-/bcryptjs-2.4.5.tgz#15473cc012f825b3435b189376f645bdd2fc9f11" + integrity sha512-tOF6TivOIvq+TWQm78335CMdyVJhpBG3NUdWQDAp95ax4E2rSKbws/ELHLk5EBoucwx/tHt3/hhLOHwWJgVrSw== "@types/body-parser@*": version "1.19.3" @@ -1361,10 +1361,10 @@ resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.3.5.tgz#ae69bcbb1bebb68c4ac0b11e9d8ed04526b3562b" integrity sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng== -"@types/cheerio@0.22.31": - version "0.22.31" - resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.31.tgz#b8538100653d6bb1b08a1e46dec75b4f2a5d5eb6" - integrity sha512-Kt7Cdjjdi2XWSfrZ53v4Of0wG3ZcmaegFXjMmz9tfNrZSkzzo36G0AL1YqSdcIA78Etjt6E609pt5h1xnQkPUw== +"@types/cheerio@0.22.33": + version "0.22.33" + resolved "https://registry.yarnpkg.com/@types/cheerio/-/cheerio-0.22.33.tgz#e4792408b107384d7d7469e3b4d31408078ec620" + integrity sha512-XUlu2BK4q3xJsccRLK69m/cABZd7m60o+cDEPUTG6jTpuG2vqN35UioeF99MQ/HoSOEPq0Bgil8g3jtzE0oH9A== dependencies: "@types/node" "*" @@ -1375,10 +1375,10 @@ dependencies: "@types/node" "*" -"@types/content-disposition@0.5.5": - version "0.5.5" - resolved "https://registry.yarnpkg.com/@types/content-disposition/-/content-disposition-0.5.5.tgz#650820e95de346e1f84e30667d168c8fd25aa6e3" - integrity sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA== +"@types/content-disposition@0.5.7": + version "0.5.7" + resolved "https://registry.yarnpkg.com/@types/content-disposition/-/content-disposition-0.5.7.tgz#3b98d4bf8c80640f93b042511acb5aad18139748" + integrity sha512-V9/5u21RHFR1zfdm3rQ6pJUKV+zSSVQt+yq16i1YhdivVzWgPEoKedc3GdT8aFjsqQbakdxuy3FnEdePUQOamQ== "@types/cookie@^0.4.1": version "0.4.1" @@ -1448,10 +1448,10 @@ resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-2.0.2.tgz#a86e00bbde8950364f8e7846687259ffcd96e8c2" integrity sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg== -"@types/is-utf8@0.2.1": - version "0.2.1" - resolved "https://registry.yarnpkg.com/@types/is-utf8/-/is-utf8-0.2.1.tgz#2cecf393ce44a73d3d052224e8375709098b4d25" - integrity sha512-4tSeTnvbhBsWZy+NTB7g3mbRDZKN0tgS199YlY2JngABhpxSlKyaUX65Lxw8VnLa6IG4tHxBMi0ffhnFhio7jw== +"@types/is-utf8@0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@types/is-utf8/-/is-utf8-0.2.2.tgz#b31ab599ead973992809b0b802ce066abbb42efd" + integrity sha512-j7PFtO0ki4IoJvmMaAHQ70z74Td244dMLC7BAz5pb0v7IC8xXLtuM+7AWsMco4Minz92m30fO72+TbkmtMr4dQ== dependencies: "@types/node" "*" @@ -1479,10 +1479,10 @@ resolved "https://registry.yarnpkg.com/@types/linkify-it/-/linkify-it-3.0.3.tgz#15a0712296c5041733c79efe233ba17ae5a7587b" integrity sha512-pTjcqY9E4nOI55Wgpz7eiI8+LzdYnw3qxXCfHyBDdPbYvbyLgWLJGh8EdPvqawwMK1Uo1794AUkkR38Fr0g+2g== -"@types/lodash@4.14.195": - version "4.14.195" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.195.tgz#bafc975b252eb6cea78882ce8a7b6bf22a6de632" - integrity sha512-Hwx9EUgdwf2GLarOjQp5ZH8ZmblzcbTBC2wtQWNKARBSxM9ezRIAUpeDTgoQRAFB0+8CNWXVA9+MaSOzOF3nPg== +"@types/lodash@4.14.200": + version "4.14.200" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.200.tgz#435b6035c7eba9cdf1e039af8212c9e9281e7149" + integrity sha512-YI/M/4HRImtNf3pJgbF+W6FrXovqj+T+/HpENLTooK9PnkacBsDpeP3IpHab40CClUfhNmdM2WTNP2sa2dni5Q== "@types/mime-types@2.1.1": version "2.1.1" @@ -1514,10 +1514,10 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-9.1.1.tgz#e7c4f1001eefa4b8afbd1eee27a237fee3bf29c4" integrity sha512-Z61JK7DKDtdKTWwLeElSEBcWGRLY8g95ic5FoQqI9CMx0ns/Ghep3B4DfcEimiKMvtamNVULVNKEsiwV3aQmXw== -"@types/mousetrap@1.6.11": - version "1.6.11" - resolved "https://registry.yarnpkg.com/@types/mousetrap/-/mousetrap-1.6.11.tgz#ef9620160fdcefcb85bccda8aaa3e84d7429376d" - integrity sha512-F0oAily9Q9QQpv9JKxKn0zMKfOo36KHCW7myYsmUyf2t0g+sBTbG3UleTPoguHdE1z3GLFr3p7/wiOio52QFjQ== +"@types/mousetrap@1.6.13": + version "1.6.13" + resolved "https://registry.yarnpkg.com/@types/mousetrap/-/mousetrap-1.6.13.tgz#1b2e4cd374fdd1ee58a240be0aafd94f7270b3be" + integrity sha512-dEzDpaR+P/thkMsjsREQDX9OP8AMyLncTkgUgTTIxq5lJTlQffiLJt67ImDtaX+kC7CaNIX30pfdrrMZkym+eg== "@types/node@*", "@types/node@>=10.0.0": version "20.8.4" @@ -2779,6 +2779,11 @@ caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001538, caniuse-lite@^1.0.30001541: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001546.tgz#10fdad03436cfe3cc632d3af7a99a0fb497407f0" integrity sha512-zvtSJwuQFpewSyRrI3AsftF6rM0X80mZkChIt1spBGEvRglCrjTniXvinc8JKRoqTwXAgvqTImaN9igfSMtUBw== +caniuse-lite@^1.0.30001561: + version "1.0.30001565" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001565.tgz#a528b253c8a2d95d2b415e11d8b9942acc100c4f" + integrity sha512-xrE//a3O7TP0vaJ8ikzkD2c2NgcVUvsEe2IvFTntV4Yd1Z9FVzh+gW+enX96L0psrbaFMcVcH2l90xNuGDWc8w== + chai@4.3.7: version "4.3.7" resolved "https://registry.yarnpkg.com/chai/-/chai-4.3.7.tgz#ec63f6df01829088e8bf55fca839bcd464a8ec51" @@ -4236,7 +4241,7 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== -get-func-name@^2.0.0, get-func-name@^2.0.2: +get-func-name@2.0.2, get-func-name@^2.0.0, get-func-name@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/get-func-name/-/get-func-name-2.0.2.tgz#0d7cf20cd13fda808669ffa88f4ffc7a3943fc41" integrity sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ== @@ -5755,9 +5760,9 @@ nanoid@3.3.1: integrity sha512-n6Vs/3KGyxPQd6uO0eH4Bv0ojGSUvuLlIHtC3Y0kEO23YRge8H9x1GCzLn28YX0H66pMkxuaeESFq4tKISKwdw== nanoid@^3.3.6: - version "3.3.6" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.6.tgz#443380c856d6e9f9824267d960b4236ad583ea4c" - integrity sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA== + version "3.3.7" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" + integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== natural-compare@^1.4.0: version "1.4.0" From 42e08dec93017f6aa7ace3b4697e4e52e0dd4e62 Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:09:53 +0000 Subject: [PATCH 2/7] CLI sqlite migratior --- server/clientManager.ts | 38 ++++++++++++------- server/command-line/index.ts | 9 +++-- server/command-line/storage.ts | 68 ++++++++++++++++++++++++++++++++++ 3 files changed, 99 insertions(+), 16 deletions(-) create mode 100644 server/command-line/storage.ts diff --git a/server/clientManager.ts b/server/clientManager.ts index 78e94d18..70b7e81b 100644 --- a/server/clientManager.ts +++ b/server/clientManager.ts @@ -5,12 +5,12 @@ import fs from "fs"; import path from "path"; import Auth from "./plugins/auth"; -import Client, {UserConfig} from "./client"; +import Client, { UserConfig } from "./client"; import Config from "./config"; -import {NetworkConfig} from "./models/network"; +import { NetworkConfig } from "./models/network"; import WebPush from "./plugins/webpush"; import log from "./log"; -import {Server} from "socket.io"; +import { Server } from "socket.io"; class ClientManager { clients: Client[]; @@ -107,17 +107,21 @@ class ClientManager { // Existing users removed since last time users were loaded _.difference(loaded, updatedUsers).forEach((name) => { - const client = _.find(this.clients, {name}); + const client = _.find(this.clients, { name }); if (client) { client.quit(true); this.clients = _.without(this.clients, client); - log.info(`User ${colors.bold(name)} disconnected and removed.`); + log.info( + `User ${colors.bold( + name + )} disconnected and removed.` + ); } }); }, 1000, - {maxWait: 10000} + { maxWait: 10000 } ) ); } @@ -197,7 +201,8 @@ class ClientManager { if ( userFolderStat && userFileStat && - (userFolderStat.uid !== userFileStat.uid || userFolderStat.gid !== userFileStat.gid) + (userFolderStat.uid !== userFileStat.uid || + userFolderStat.gid !== userFileStat.gid) ) { log.warn( `User ${colors.green( @@ -227,13 +232,16 @@ class ClientManager { networks: client.networks.map((n) => n.export()), }); const newUser = JSON.stringify(json, null, "\t"); - const newHash = crypto.createHash("sha256").update(newUser).digest("hex"); + const newHash = crypto + .createHash("sha256") + .update(newUser) + .digest("hex"); - return {newUser, newHash}; + return { newUser, newHash }; } saveUser(client: Client, callback?: (err?: any) => void) { - const {newUser, newHash} = this.getDataToSave(client); + const { newUser, newHash } = this.getDataToSave(client); // Do not write to disk if the exported data hasn't actually changed if (client.fileHash === newHash) { @@ -254,7 +262,9 @@ class ClientManager { return callback ? callback() : true; } catch (e: any) { // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - log.error(`Failed to update user ${colors.green(client.name)} (${e})`); + log.error( + `Failed to update user ${colors.green(client.name)} (${e})` + ); if (callback) { callback(e); @@ -266,7 +276,9 @@ class ClientManager { const userPath = Config.getUserConfigPath(name); if (!fs.existsSync(userPath)) { - log.error(`Tried to remove non-existing user ${colors.green(name)}.`); + log.error( + `Tried to remove non-existing user ${colors.green(name)}.` + ); return false; } @@ -275,7 +287,7 @@ class ClientManager { return true; } - private readUserConfig(name: string) { + readUserConfig(name: string) { const userPath = Config.getUserConfigPath(name); if (!fs.existsSync(userPath)) { diff --git a/server/command-line/index.ts b/server/command-line/index.ts index 44d79a2a..55a6320a 100644 --- a/server/command-line/index.ts +++ b/server/command-line/index.ts @@ -3,7 +3,7 @@ import log from "../log"; import fs from "fs"; import path from "path"; import colors from "chalk"; -import {Command} from "commander"; +import { Command } from "commander"; import Helper from "../helper"; import Config from "../config"; import Utils from "./utils"; @@ -42,6 +42,7 @@ program.addCommand(require("./install").default); program.addCommand(require("./uninstall").default); program.addCommand(require("./upgrade").default); program.addCommand(require("./outdated").default); +program.addCommand(require("./storage").default); if (!Config.values.public) { require("./users").default.forEach((command: Command) => { @@ -64,7 +65,7 @@ function createPackagesFolder() { const packagesConfig = path.join(packagesPath, "package.json"); // Create node_modules folder, otherwise yarn will start walking upwards to find one - fs.mkdirSync(path.join(packagesPath, "node_modules"), {recursive: true}); + fs.mkdirSync(path.join(packagesPath, "node_modules"), { recursive: true }); // Create package.json with private set to true, if it doesn't exist already if (!fs.existsSync(packagesConfig)) { @@ -99,7 +100,9 @@ function verifyFileOwner() { ); } - const configStat = fs.statSync(path.join(Config.getHomePath(), "config.js")); + const configStat = fs.statSync( + path.join(Config.getHomePath(), "config.js") + ); if (configStat && configStat.uid !== uid) { log.warn( diff --git a/server/command-line/storage.ts b/server/command-line/storage.ts new file mode 100644 index 00000000..4848f326 --- /dev/null +++ b/server/command-line/storage.ts @@ -0,0 +1,68 @@ +import log from "../log"; +import { Command } from "commander"; +import ClientManager from "../clientManager"; +import Utils from "./utils"; +import SqliteMessageStorage from "../plugins/messageStorage/sqlite"; + +const program = new Command("storage").description( + "various utilities related to the message storage" +); + +program + .command("migrate") + .argument("[user]", "migrate a specific user only, all if not provided") + .description("Migrate message storage where needed") + .on("--help", Utils.extraHelp) + .action(function (user) { + runMigrations(user).catch((err) => { + log.error(err.toString()); + process.exit(1); + }); + }); + +async function runMigrations(user: string) { + const manager = new ClientManager(); + const users = manager.getUsers(); + + if (user) { + if (!users.includes(user)) { + throw new Error(`invalid user ${user}`); + } + + return migrateUser(manager, user); + } + + for (const name of users) { + await migrateUser(manager, name); + // if any migration fails we blow up, + // chances are the rest won't complete either + } +} + +// runs sqlite migrations for a user, which must exist +async function migrateUser(manager: ClientManager, user: string) { + log.info("handling user", user); + + if (!isUserLogEnabled(manager, user)) { + log.info("logging disabled for user", user, ". Skipping"); + return; + } + + const sqlite = new SqliteMessageStorage(user); + await sqlite.enable(); // enable runs migrations + await sqlite.close(); + log.info("user", user, "migrated successfully"); +} + +function isUserLogEnabled(manager: ClientManager, user: string): boolean { + const conf = manager.readUserConfig(user); + + if (!conf) { + log.error("Could not open user configuration of", user); + return false; + } + + return conf.log; +} + +export default program; From 59cc53014d5f565e71d589e4d573350abf2b884a Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:21:56 +0000 Subject: [PATCH 3/7] Prevent crashing on rDNS failure --- server/server.ts | 255 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 176 insertions(+), 79 deletions(-) diff --git a/server/server.ts b/server/server.ts index 4aecc5e1..215a6848 100644 --- a/server/server.ts +++ b/server/server.ts @@ -1,9 +1,9 @@ import _ from "lodash"; -import {Server as wsServer} from "ws"; -import express, {NextFunction, Request, Response} from "express"; +import { Server as wsServer } from "ws"; +import express, { NextFunction, Request, Response } from "express"; import fs from "fs"; import path from "path"; -import {Server, Socket} from "socket.io"; +import { Server, Socket } from "socket.io"; import dns from "dns"; import colors from "chalk"; import net from "net"; @@ -13,18 +13,18 @@ import Client from "./client"; import ClientManager from "./clientManager"; import Uploader from "./plugins/uploader"; import Helper from "./helper"; -import Config, {ConfigType, Defaults} from "./config"; +import Config, { ConfigType, Defaults } from "./config"; import Identification from "./identification"; import changelog from "./plugins/changelog"; import inputs from "./plugins/inputs"; import Auth from "./plugins/auth"; -import themes, {ThemeForClient} from "./plugins/packages/themes"; +import themes, { ThemeForClient } from "./plugins/packages/themes"; themes.loadLocalThemes(); import packages from "./plugins/packages/index"; -import {NetworkWithIrcFramework} from "./models/network"; -import {ChanType} from "./models/chan"; +import { NetworkWithIrcFramework } from "./models/network"; +import { ChanType } from "./models/chan"; import Utils from "./command-line/utils"; import type { ClientToServerEvents, @@ -72,9 +72,9 @@ export default async function ( } ) { log.info(`Hard Lounge ${colors.green(Helper.getVersion())} \ -(Node.js ${colors.green(process.versions.node)} on ${colors.green(process.platform)} ${ - process.arch - })`); +(Node.js ${colors.green(process.versions.node)} on ${colors.green( + process.platform + )} ${process.arch})`); log.info(`Configuration file: ${colors.green(Config.getConfigPath())}`); const staticOptions = { @@ -96,8 +96,16 @@ export default async function ( .get("/service-worker.js", forceNoCacheRequest) .get("/js/bundle.js.map", forceNoCacheRequest) .get("/css/style.css.map", forceNoCacheRequest) - .use(express.static(Utils.getFileFromRelativeToRoot("public"), staticOptions)) - .use("/storage/", express.static(Config.getStoragePath(), staticOptions)); + .use( + express.static( + Utils.getFileFromRelativeToRoot("public"), + staticOptions + ) + ) + .use( + "/storage/", + express.static(Config.getStoragePath(), staticOptions) + ); if (Config.values.fileUpload.enable) { Uploader.router(app); @@ -123,7 +131,10 @@ export default async function ( const fileName = req.params.filename; const packageFile = packages.getPackage(packageName); - if (!packageFile || !packages.getFiles().includes(`${packageName}/${fileName}`)) { + if ( + !packageFile || + !packages.getFiles().includes(`${packageName}/${fileName}`) + ) { return res.status(404).send("Not found"); } @@ -180,7 +191,10 @@ export default async function ( host: string | undefined; }; - if (typeof Config.values.host === "string" && Config.values.host.startsWith("unix:")) { + if ( + typeof Config.values.host === "string" && + Config.values.host.startsWith("unix:") + ) { listenParams = Config.values.host.replace(/^unix:/, ""); } else { listenParams = { @@ -208,8 +222,12 @@ export default async function ( log.info( "Available at " + - colors.green(`${protocol}://${address.address}:${address.port}/`) + - ` in ${colors.bold(Config.values.public ? "public" : "private")} mode` + colors.green( + `${protocol}://${address.address}:${address.port}/` + ) + + ` in ${colors.bold( + Config.values.public ? "public" : "private" + )} mode` ); } } @@ -267,7 +285,9 @@ export default async function ( log.error(`Could not start identd server, ${err.message}`); process.exit(1); } else if (!manager) { - log.error("Could not start identd server, ClientManager is undefined"); + log.error( + "Could not start identd server, ClientManager is undefined" + ); process.exit(1); } @@ -290,7 +310,9 @@ export default async function ( } if (Config.values.prefetchStorage) { - log.info("Clearing prefetch storage folder, this might take a while..."); + log.info( + "Clearing prefetch storage folder, this might take a while..." + ); (await import("./plugins/storage")).default.emptyDir(); } @@ -316,7 +338,7 @@ export default async function ( // Clear storage folder after server starts successfully if (Config.values.prefetchStorage) { import("./plugins/storage") - .then(({default: storage}) => { + .then(({ default: storage }) => { storage.emptyDir(); }) .catch((err: Error) => { @@ -333,7 +355,10 @@ export default async function ( function getClientLanguage(socket: Socket): string | null { const acceptLanguage = socket.handshake.headers["accept-language"]; - if (typeof acceptLanguage === "string" && /^[\x00-\x7F]{1,50}$/.test(acceptLanguage)) { + if ( + typeof acceptLanguage === "string" && + /^[\x00-\x7F]{1,50}$/.test(acceptLanguage) + ) { // only allow ASCII strings between 1-50 characters in length return acceptLanguage; } @@ -360,7 +385,10 @@ function getClientIp(socket: Socket) { function getClientSecure(socket: Socket) { let secure = socket.handshake.secure; - if (Config.values.reverseProxy && socket.handshake.headers["x-forwarded-proto"] === "https") { + if ( + Config.values.reverseProxy && + socket.handshake.headers["x-forwarded-proto"] === "https" + ) { secure = true; } @@ -390,7 +418,9 @@ function addSecurityHeaders(req: Request, res: Response, next: NextFunction) { // - https://user-images.githubusercontent.com is where we currently push our changelog screenshots // - data: is required for the HTML5 video player if (Config.values.prefetchStorage || !Config.values.prefetch) { - policies.push("img-src 'self' data: https://user-images.githubusercontent.com"); + policies.push( + "img-src 'self' data: https://user-images.githubusercontent.com" + ); policies.unshift("block-all-mixed-content"); } else { policies.push("img-src http: https: data:"); @@ -422,7 +452,7 @@ function indexRequest(req: Request, res: Response) { const config: IndexTemplateConfiguration = { ...getServerConfiguration(), - ...{cacheBust: Helper.getVersionCacheBust()}, + ...{ cacheBust: Helper.getVersionCacheBust() }, }; res.send(_.template(file)(config)); @@ -449,7 +479,9 @@ function initializeClient( // If client provided channel passes checks, use it. if client has invalid // channel open (or windows like settings) then use last known server active channel - openChannel = client.attachedClients[socket.id].openChannel || client.lastActiveChannel; + openChannel = + client.attachedClients[socket.id].openChannel || + client.lastActiveChannel; } else { openChannel = client.lastActiveChannel; } @@ -494,7 +526,7 @@ function initializeClient( return; } - const network = _.find(client.networks, {uuid: data}); + const network = _.find(client.networks, { uuid: data }); if (!network) { return; @@ -508,7 +540,7 @@ function initializeClient( return; } - const network = _.find(client.networks, {uuid: data.uuid}); + const network = _.find(client.networks, { uuid: data.uuid }); if (!network) { return; @@ -552,7 +584,10 @@ function initializeClient( const hash = Helper.password.hash(p1); client.setPassword(hash, (success: boolean) => { - const obj = {success: false, error: undefined} as { + const obj = { + success: false, + error: undefined, + } as { success: boolean; error: string | undefined; }; @@ -567,7 +602,9 @@ function initializeClient( }); }) .catch((error: Error) => { - log.error(`Error while checking users password. Error: ${error.message}`); + log.error( + `Error while checking users password. Error: ${error.message}` + ); }); } }); @@ -596,7 +633,9 @@ function initializeClient( socket.emit("changelog", changelogData); }) .catch((error: Error) => { - log.error(`Error while fetching changelog. Error: ${error.message}`); + log.error( + `Error while fetching changelog. Error: ${error.message}` + ); }); }); @@ -665,7 +704,12 @@ function initializeClient( if (!Config.values.public) { socket.on("push:register", (subscription) => { - if (!Object.prototype.hasOwnProperty.call(client.config.sessions, token)) { + if ( + !Object.prototype.hasOwnProperty.call( + client.config.sessions, + token + ) + ) { return; } @@ -684,26 +728,32 @@ function initializeClient( } }); - socket.on("push:unregister", () => client.unregisterPushSubscription(token)); + socket.on("push:unregister", () => + client.unregisterPushSubscription(token) + ); } const sendSessionList = () => { // TODO: this should use the ClientSession type currently in client - const sessions = _.map(client.config.sessions, (session, sessionToken) => { - return { - current: sessionToken === token, - active: _.reduce( - client.attachedClients, - (count, attachedClient) => - count + (attachedClient.token === sessionToken ? 1 : 0), - 0 - ), - lastUse: session.lastUse, - ip: session.ip, - agent: session.agent, - token: sessionToken, // TODO: Ideally don't expose actual tokens to the client - }; - }); + const sessions = _.map( + client.config.sessions, + (session, sessionToken) => { + return { + current: sessionToken === token, + active: _.reduce( + client.attachedClients, + (count, attachedClient) => + count + + (attachedClient.token === sessionToken ? 1 : 0), + 0 + ), + lastUse: session.lastUse, + ip: session.ip, + agent: session.agent, + token: sessionToken, // TODO: Ideally don't expose actual tokens to the client + }; + } + ); socket.emit("sessions:list", sessions); }; @@ -725,8 +775,12 @@ function initializeClient( } // We do not need to do write operations and emit events if nothing changed. - if (client.config.clientSettings[newSetting.name] !== newSetting.value) { - client.config.clientSettings[newSetting.name] = newSetting.value; + if ( + client.config.clientSettings[newSetting.name] !== + newSetting.value + ) { + client.config.clientSettings[newSetting.name] = + newSetting.value; // Pass the setting to all clients. client.emit("setting:new", { @@ -736,7 +790,10 @@ function initializeClient( client.save(); - if (newSetting.name === "highlights" || newSetting.name === "highlightExceptions") { + if ( + newSetting.name === "highlights" || + newSetting.name === "highlightExceptions" + ) { client.compileCustomHighlights(); } else if (newSetting.name === "awayMessage") { if (typeof newSetting.value !== "string") { @@ -749,7 +806,12 @@ function initializeClient( }); socket.on("setting:get", () => { - if (!Object.prototype.hasOwnProperty.call(client.config, "clientSettings")) { + if ( + !Object.prototype.hasOwnProperty.call( + client.config, + "clientSettings" + ) + ) { socket.emit("setting:all", {}); return; } @@ -763,14 +825,14 @@ function initializeClient( socket.emit("search:results", results); }); - socket.on("mute:change", ({target, setMutedTo}) => { + socket.on("mute:change", ({ target, setMutedTo }) => { const networkAndChan = client.find(target); if (!networkAndChan) { return; } - const {chan, network} = networkAndChan; + const { chan, network } = networkAndChan; // If the user mutes the lobby, we mute the entire network. if (chan.type === ChanType.LOBBY) { @@ -802,7 +864,12 @@ function initializeClient( tokenToSignOut = token; } - if (!Object.prototype.hasOwnProperty.call(client.config.sessions, tokenToSignOut)) { + if ( + !Object.prototype.hasOwnProperty.call( + client.config.sessions, + tokenToSignOut + ) + ) { return; } @@ -815,7 +882,9 @@ function initializeClient( return; } - const socketToRemove = manager!.sockets.of("/").sockets.get(socketId); + const socketToRemove = manager!.sockets + .of("/") + .sockets.get(socketId); socketToRemove!.emit("sign-out"); socketToRemove!.disconnect(); @@ -901,7 +970,7 @@ function getClientConfiguration(): ClientConfiguration { } function getServerConfiguration(): ServerConfiguration { - return {...Config.values, ...{stylesheets: packages.getStylesheets()}}; + return { ...Config.values, ...{ stylesheets: packages.getStylesheets() } }; } function performAuthentication(this: Socket, data) { @@ -914,7 +983,13 @@ function performAuthentication(this: Socket, data) { let token: string; const finalInit = () => - initializeClient(socket, client, token, data.lastMessage || -1, data.openChannel); + initializeClient( + socket, + client, + token, + data.lastMessage || -1, + data.openChannel + ); const initClient = () => { // Configuration does not change during runtime of TL, @@ -924,7 +999,9 @@ function performAuthentication(this: Socket, data) { socket.emit( "push:issubscribed", - token && client.config.sessions[token].pushSubscription ? true : false + token && client.config.sessions[token].pushSubscription + ? true + : false ); } @@ -976,9 +1053,9 @@ function performAuthentication(this: Socket, data) { ); } else { log.warn( - `Authentication failed for user ${colors.bold(data.user)} from ${colors.bold( - getClientIp(socket) - )}` + `Authentication failed for user ${colors.bold( + data.user + )} from ${colors.bold(getClientIp(socket))}` ); } @@ -1001,7 +1078,12 @@ function performAuthentication(this: Socket, data) { if (client && data.token) { const providedToken = client.calculateTokenHash(data.token); - if (Object.prototype.hasOwnProperty.call(client.config.sessions, providedToken)) { + if ( + Object.prototype.hasOwnProperty.call( + client.config.sessions, + providedToken + ) + ) { token = providedToken; return authCallback(true); @@ -1015,28 +1097,43 @@ function performAuthentication(this: Socket, data) { } function reverseDnsLookup(ip: string, callback: (hostname: string) => void) { - dns.reverse(ip, (reverseErr, hostnames) => { - if (reverseErr || hostnames.length < 1) { - return callback(ip); - } - - dns.resolve(hostnames[0], net.isIP(ip) === 6 ? "AAAA" : "A", (resolveErr, resolvedIps) => { - // TODO: investigate SoaRecord class - if (!Array.isArray(resolvedIps)) { + // node can throw, even if we provide valid input based on the DNS server + // returning SERVFAIL it seems: https://github.com/thelounge/thelounge/issues/4768 + // so we manually resolve with the ip as a fallback in case something fails + try { + dns.reverse(ip, (reverseErr, hostnames) => { + if (reverseErr || hostnames.length < 1) { return callback(ip); } - if (resolveErr || resolvedIps.length < 1) { - return callback(ip); - } + dns.resolve( + hostnames[0], + net.isIP(ip) === 6 ? "AAAA" : "A", + (resolveErr, resolvedIps) => { + // TODO: investigate SoaRecord class + if (!Array.isArray(resolvedIps)) { + return callback(ip); + } - for (const resolvedIp of resolvedIps) { - if (ip === resolvedIp) { - return callback(hostnames[0]); + if (resolveErr || resolvedIps.length < 1) { + return callback(ip); + } + + for (const resolvedIp of resolvedIps) { + if (ip === resolvedIp) { + return callback(hostnames[0]); + } + } + + return callback(ip); } - } - - return callback(ip); + ); }); - }); + } catch (err) { + log.error( + `failed to resolve rDNS for ${ip}, using ip instead`, + (err as any).toString() + ); + setImmediate(callback, ip); // makes sure we always behave asynchronously + } } From e91585f9abb8d25e1a1bd6591d2c397c0a5e505d Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:23:14 +0000 Subject: [PATCH 4/7] Fix fetch_rollbacks typo --- server/plugins/messageStorage/sqlite.ts | 65 ++++++++++++++++--------- 1 file changed, 43 insertions(+), 22 deletions(-) diff --git a/server/plugins/messageStorage/sqlite.ts b/server/plugins/messageStorage/sqlite.ts index 880abf06..8961e2d9 100644 --- a/server/plugins/messageStorage/sqlite.ts +++ b/server/plugins/messageStorage/sqlite.ts @@ -1,13 +1,17 @@ -import type {Database} from "sqlite3"; +import type { Database } from "sqlite3"; import log from "../../log"; import path from "path"; import fs from "fs/promises"; import Config from "../../config"; -import Msg, {Message} from "../../models/msg"; -import Chan, {Channel} from "../../models/chan"; +import Msg, { Message } from "../../models/msg"; +import Chan, { Channel } from "../../models/chan"; import Helper from "../../helper"; -import type {SearchResponse, SearchQuery, SearchableMessageStorage} from "./types"; +import type { + SearchResponse, + SearchQuery, + SearchableMessageStorage, +} from "./types"; import Network from "../../models/network"; // TODO; type @@ -16,15 +20,21 @@ let sqlite3: any; try { sqlite3 = require("sqlite3"); } catch (e: any) { - Config.values.messageStorage = Config.values.messageStorage.filter((item) => item !== "sqlite"); + Config.values.messageStorage = Config.values.messageStorage.filter( + (item) => item !== "sqlite" + ); log.error( "Unable to load sqlite3 module. See https://github.com/mapbox/node-sqlite3/wiki/Binaries" ); } -type Migration = {version: number; stmts: string[]}; -type Rollback = {version: number; rollback_forbidden?: boolean; stmts: string[]}; +type Migration = { version: number; stmts: string[] }; +type Rollback = { + version: number; + rollback_forbidden?: boolean; + stmts: string[]; +}; export const currentSchemaVersion = 1679743888000; // use `new Date().getTime()` @@ -121,7 +131,7 @@ class SqliteMessageStorage implements SearchableMessageStorage { const sqlitePath = path.join(logsPath, `${this.userName}.sqlite3`); try { - await fs.mkdir(logsPath, {recursive: true}); + await fs.mkdir(logsPath, { recursive: true }); } catch (e) { throw Helper.catch_to_error("Unable to create logs directory", e); } @@ -152,9 +162,10 @@ class SqliteMessageStorage implements SearchableMessageStorage { await this.serialize_run(stmt, []); } - await this.serialize_run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", [ - currentSchemaVersion.toString(), - ]); + await this.serialize_run( + "INSERT INTO options (name, value) VALUES ('schema_version', ?)", + [currentSchemaVersion.toString()] + ); } async current_version(): Promise { @@ -181,9 +192,10 @@ class SqliteMessageStorage implements SearchableMessageStorage { } async update_version_in_db() { - return this.serialize_run("UPDATE options SET value = ? WHERE name = 'schema_version'", [ - currentSchemaVersion.toString(), - ]); + return this.serialize_run( + "UPDATE options SET value = ? WHERE name = 'schema_version'", + [currentSchemaVersion.toString()] + ); } async _run_migrations(dbVersion: number) { @@ -274,7 +286,7 @@ class SqliteMessageStorage implements SearchableMessageStorage { stmts: [raw.statement], }); } else { - last.stmts.push(raw.statment); + last.stmts.push(raw.statement); } } @@ -282,7 +294,10 @@ class SqliteMessageStorage implements SearchableMessageStorage { } async delete_migrations_older_than(version: number) { - return this.serialize_run("delete from migrations where migrations.version > ?", [version]); + return this.serialize_run( + "delete from migrations where migrations.version > ?", + [version] + ); } async _downgrade_to(version: number) { @@ -372,7 +387,12 @@ class SqliteMessageStorage implements SearchableMessageStorage { // id is regenerated when messages are retrieved // previews are not stored because storage is cleared on lounge restart // type and time are stored in a separate column - if (prop !== "id" && prop !== "previews" && prop !== "type" && prop !== "time") { + if ( + prop !== "id" && + prop !== "previews" && + prop !== "type" && + prop !== "time" + ) { newMsg[prop] = msg[prop]; } @@ -398,10 +418,10 @@ class SqliteMessageStorage implements SearchableMessageStorage { return; } - await this.serialize_run("DELETE FROM messages WHERE network = ? AND channel = ?", [ - network.uuid, - channel.name.toLowerCase(), - ]); + await this.serialize_run( + "DELETE FROM messages WHERE network = ? AND channel = ?", + [network.uuid, channel.name.toLowerCase()] + ); } async getMessages( @@ -416,7 +436,8 @@ class SqliteMessageStorage implements SearchableMessageStorage { } // If unlimited history is specified, load 100k messages - const limit = Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory; + const limit = + Config.values.maxHistory < 0 ? 100000 : Config.values.maxHistory; const rows = await this.serialize_fetchall( "SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", From 01ab326e523ee8e9664c6507654f1b587f7d22ff Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:24:41 +0000 Subject: [PATCH 5/7] Prevent modifying global array during sqlite testing --- test/plugins/sqlite.ts | 53 +++++++++++++++++++++++++++--------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/test/plugins/sqlite.ts b/test/plugins/sqlite.ts index 1ce745cd..5f3ecc84 100644 --- a/test/plugins/sqlite.ts +++ b/test/plugins/sqlite.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unsafe-return */ import fs from "fs"; import path from "path"; -import {expect} from "chai"; +import { expect } from "chai"; import util from "../util"; -import Msg, {MessageType} from "../../server/models/msg"; +import Msg, { MessageType } from "../../server/models/msg"; import Config from "../../server/config"; import MessageStorage, { currentSchemaVersion, @@ -111,7 +111,7 @@ describe("SQLite migrations", function () { it("has working down-migrations", async function () { await serialize_run("BEGIN EXCLUSIVE TRANSACTION"); - for (const rollback of rollbacks.reverse()) { + for (const rollback of rollbacks.slice().reverse()) { if (rollback.rollback_forbidden) { throw Error( "Try to write a down migration, if you really can't, flip this to a break" @@ -132,7 +132,11 @@ describe("SQLite Message Storage", function () { this.timeout(util.isRunningOnCI() ? 25000 : 5000); this.slow(300); - const expectedPath = path.join(Config.getHomePath(), "logs", "testUser.sqlite3"); + const expectedPath = path.join( + Config.getHomePath(), + "logs", + "testUser.sqlite3" + ); let store: MessageStorage; function db_get_one(stmt: string, ...params: any[]): Promise { @@ -193,13 +197,19 @@ describe("SQLite Message Storage", function () { it("should resolve an empty array when disabled", async function () { store.isEnabled = false; - const messages = await store.getMessages(null as any, null as any, null as any); + const messages = await store.getMessages( + null as any, + null as any, + null as any + ); expect(messages).to.be.empty; store.isEnabled = true; }); it("should insert schema version to options table", async function () { - const row = await db_get_one("SELECT value FROM options WHERE name = 'schema_version'"); + const row = await db_get_one( + "SELECT value FROM options WHERE name = 'schema_version'" + ); expect(row.value).to.equal(currentSchemaVersion.toString()); }); @@ -252,8 +262,8 @@ describe("SQLite Message Storage", function () { for (let i = 0; i < 200; ++i) { await store.index( - {uuid: "retrieval-order-test-network"} as any, - {name: "#channel"} as any, + { uuid: "retrieval-order-test-network" } as any, + { name: "#channel" } as any, new Msg({ time: 123456789 + i, text: `msg ${i}`, @@ -263,12 +273,15 @@ describe("SQLite Message Storage", function () { let msgId = 0; const messages = await store.getMessages( - {uuid: "retrieval-order-test-network"} as any, - {name: "#channel"} as any, + { uuid: "retrieval-order-test-network" } as any, + { name: "#channel" } as any, () => msgId++ ); expect(messages).to.have.lengthOf(2); - expect(messages.map((i_1) => i_1.text)).to.deep.equal(["msg 198", "msg 199"]); + expect(messages.map((i_1) => i_1.text)).to.deep.equal([ + "msg 198", + "msg 199", + ]); } finally { Config.values.maxHistory = originalMaxHistory; } @@ -293,7 +306,9 @@ describe("SQLite Message Storage", function () { expectedMessages.push(`msg ${i}`); } - expect(search.results.map((i_1) => i_1.text)).to.deep.equal(expectedMessages); + expect(search.results.map((i_1) => i_1.text)).to.deep.equal( + expectedMessages + ); } finally { Config.values.maxHistory = originalMaxHistory; } @@ -316,8 +331,8 @@ describe("SQLite Message Storage", function () { Config.values.maxHistory = 3; await store.index( - {uuid: "this-is-a-network-guid2"} as any, - {name: "#channel"} as any, + { uuid: "this-is-a-network-guid2" } as any, + { name: "#channel" } as any, new Msg({ time: 123456790, text: `foo % bar _ baz`, @@ -325,8 +340,8 @@ describe("SQLite Message Storage", function () { ); await store.index( - {uuid: "this-is-a-network-guid2"} as any, - {name: "#channel"} as any, + { uuid: "this-is-a-network-guid2" } as any, + { name: "#channel" } as any, new Msg({ time: 123456791, text: `foo bar x baz`, @@ -334,8 +349,8 @@ describe("SQLite Message Storage", function () { ); await store.index( - {uuid: "this-is-a-network-guid2"} as any, - {name: "#channel"} as any, + { uuid: "this-is-a-network-guid2" } as any, + { name: "#channel" } as any, new Msg({ time: 123456792, text: `bar @ baz`, @@ -356,7 +371,7 @@ describe("SQLite Message Storage", function () { }); it("should be able to downgrade", async function () { - for (const rollback of rollbacks.reverse()) { + for (const rollback of rollbacks.slice().reverse()) { if (rollback.rollback_forbidden) { throw Error( "Try to write a down migration, if you really can't, flip this to a break" From 90f53da088cb15ca6b0894044f538bc1369425fa Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:26:10 +0000 Subject: [PATCH 6/7] Return new version in downgrade() --- server/plugins/messageStorage/sqlite.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/plugins/messageStorage/sqlite.ts b/server/plugins/messageStorage/sqlite.ts index 8961e2d9..676e4efb 100644 --- a/server/plugins/messageStorage/sqlite.ts +++ b/server/plugins/messageStorage/sqlite.ts @@ -322,7 +322,7 @@ class SqliteMessageStorage implements SearchableMessageStorage { await this.delete_migrations_older_than(version); await this.update_version_in_db(); - return _rollbacks.at(-1)!.version; // assert valid due to length guard above + return version; } async downgrade_to(version: number) { From 66e5bc1745fc1366b89e96ebcb7c6c45970fc5f1 Mon Sep 17 00:00:00 2001 From: hgw Date: Sun, 3 Dec 2023 02:31:32 +0000 Subject: [PATCH 7/7] Bump version to v4.4.1-1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a05b558f..29f6e1a1 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@supernets/hardlounge", "description": "The self-hosted Web IRC client", - "version": "4.4.1", + "version": "4.4.1-1", "preferGlobal": true, "bin": { "hardlounge": "index.js"