From 941849eaa8f15b5c4db7ac1bb516c161664944ac Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 28 Nov 2017 19:56:53 +0200 Subject: [PATCH] Add message indexing --- defaults/config.js | 13 +++ package.json | 1 + src/client.js | 13 ++- src/models/chan.js | 79 +++++++++++++++---- src/models/msg.js | 1 + src/plugins/inputs/query.js | 1 + src/plugins/irc-events/join.js | 2 + src/plugins/irc-events/message.js | 1 + src/plugins/irc-events/whois.js | 1 + src/plugins/sqlite.js | 126 ++++++++++++++++++++++++++++++ test/fixtures/.gitignore | 1 + yarn.lock | 31 +++++--- 12 files changed, 243 insertions(+), 27 deletions(-) create mode 100644 src/plugins/sqlite.js diff --git a/defaults/config.js b/defaults/config.js index ad53a387..ce5f5af2 100644 --- a/defaults/config.js +++ b/defaults/config.js @@ -148,6 +148,19 @@ module.exports = { // @default null webirc: null, + // + // Message logging + // Logging is also controlled per user individually (logs variable) + // Leave the array empty to disable all logging globally + // + // text: Text file per network/channel in user folder + // sqlite: Messages are stored in SQLite, this allows them to be reloaded on server restart + // + // @type array + // @default ["sqlite", "text"] + // + messageStorage: ["sqlite", "text"], + // // Log settings // diff --git a/package.json b/package.json index 732e0ffc..c8108548 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "semver": "5.5.0", "socket.io": "2.0.4", "spdy": "3.4.7", + "sqlite3": "3.1.13", "thelounge-ldapjs-non-maintained-fork": "1.0.2", "ua-parser-js": "0.7.17", "urijs": "1.19.1", diff --git a/src/client.js b/src/client.js index 980c9d49..c2264732 100644 --- a/src/client.js +++ b/src/client.js @@ -10,6 +10,7 @@ const Network = require("./models/network"); const ircFramework = require("irc-framework"); const Helper = require("./helper"); const UAParser = require("ua-parser-js"); +const MessageStorage = require("./plugins/sqlite"); module.exports = Client; @@ -78,8 +79,16 @@ function Client(manager, name, config = {}) { }); const client = this; - let delay = 0; + + if (!Helper.config.public) { + client.messageStorage = new MessageStorage(); + + if (client.config.log && Helper.config.messageStorage.includes("sqlite")) { + client.messageStorage.enable(client.name); + } + } + (client.config.networks || []).forEach((n) => { setTimeout(function() { client.connect(n); @@ -274,6 +283,8 @@ Client.prototype.connect = function(args) { network.irc.connect(); client.save(); + + channels.forEach((channel) => channel.loadMessages(client, network)); }; Client.prototype.generateToken = function(callback) { diff --git a/src/models/chan.js b/src/models/chan.js index b2ab9cc0..446ef8e4 100644 --- a/src/models/chan.js +++ b/src/models/chan.js @@ -63,11 +63,7 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) { return; } - this.messages.push(msg); - - if (client.config.log === true) { - writeUserLog.call(this, client, msg); - } + this.writeUserLog(client, msg); if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) { const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory); @@ -183,21 +179,72 @@ Chan.prototype.getFilteredClone = function(lastActiveChannel, lastMessage) { }, {}); }; -function writeUserLog(client, msg) { - if (!msg.isLoggable()) { - return false; +Chan.prototype.writeUserLog = function(client, msg) { + this.messages.push(msg); + + // Does this user have logs disabled + if (!client.config.log) { + return; } + // Are logs disabled server-wide + if (Helper.config.messageStorage.length === 0) { + return; + } + + // Is this particular message or channel loggable + if (!msg.isLoggable() || !this.isLoggable()) { + return; + } + + // Find the parent network where this channel is in const target = client.find(this.id); if (!target) { - return false; + return; } - userLog.write( - client.name, - target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs - this.type === Chan.Type.LOBBY ? target.network.host : this.name, - msg - ); -} + // TODO: Something more pluggable + if (Helper.config.messageStorage.includes("sqlite")) { + client.messageStorage.index(target.network.uuid, this.name, msg); + } + + if (Helper.config.messageStorage.includes("text")) { + userLog.write( + client.name, + target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs + this.type === Chan.Type.LOBBY ? target.network.host : this.name, + msg + ); + } +}; + +Chan.prototype.loadMessages = function(client, network) { + if (!client.messageStorage || !this.isLoggable()) { + return; + } + + client.messageStorage + .getMessages(network, this) + .then((messages) => { + if (messages.length === 0) { + return; + } + + this.messages.unshift(...messages); + + if (!this.firstUnread) { + this.firstUnread = messages[messages.length - 1].id; + } + + client.emit("more", { + chan: this.id, + messages: messages.slice(-100), + }); + }) + .catch((err) => log.error(`Failed to load messages: ${err}`)); +}; + +Chan.prototype.isLoggable = function() { + return this.type === Chan.Type.CHANNEL || this.type === Chan.Type.QUERY; +}; diff --git a/src/models/msg.js b/src/models/msg.js index e1b7dc72..a91bb76b 100644 --- a/src/models/msg.js +++ b/src/models/msg.js @@ -40,6 +40,7 @@ class Msg { isLoggable() { return this.type !== Msg.Type.MOTD && + this.type !== Msg.Type.ERROR && this.type !== Msg.Type.BANLIST && this.type !== Msg.Type.WHOIS; } diff --git a/src/plugins/inputs/query.js b/src/plugins/inputs/query.js index 8a4fb910..7c7875c1 100644 --- a/src/plugins/inputs/query.js +++ b/src/plugins/inputs/query.js @@ -54,4 +54,5 @@ exports.input = function(network, chan, cmd, args) { shouldOpen: true, }); this.save(); + newChan.loadMessages(this, network); }; diff --git a/src/plugins/irc-events/join.js b/src/plugins/irc-events/join.js index f889f51d..23d1b7e0 100644 --- a/src/plugins/irc-events/join.js +++ b/src/plugins/irc-events/join.js @@ -22,6 +22,8 @@ module.exports = function(irc, network) { chan: chan.getFilteredClone(true), }); + chan.loadMessages(client, network); + // Request channels' modes network.irc.raw("MODE", chan.name); } else if (data.nick === irc.user.nick) { diff --git a/src/plugins/irc-events/message.js b/src/plugins/irc-events/message.js index 06b063fc..35b8bdc1 100644 --- a/src/plugins/irc-events/message.js +++ b/src/plugins/irc-events/message.js @@ -72,6 +72,7 @@ module.exports = function(irc, network) { network: network.id, chan: chan.getFilteredClone(true), }); + chan.loadMessages(client, network); } } diff --git a/src/plugins/irc-events/whois.js b/src/plugins/irc-events/whois.js index d0b8a93a..cdc02a7e 100644 --- a/src/plugins/irc-events/whois.js +++ b/src/plugins/irc-events/whois.js @@ -19,6 +19,7 @@ module.exports = function(irc, network) { network: network.id, chan: chan.getFilteredClone(true), }); + chan.loadMessages(client, network); } let msg; diff --git a/src/plugins/sqlite.js b/src/plugins/sqlite.js new file mode 100644 index 00000000..a00dd29e --- /dev/null +++ b/src/plugins/sqlite.js @@ -0,0 +1,126 @@ +"use strict"; + +const path = require("path"); +const fsextra = require("fs-extra"); +const sqlite3 = require("sqlite3"); +const Helper = require("../helper"); +const Msg = require("../models/msg"); + +const currentSchemaVersion = 1; + +const schema = [ + // Schema version #1 + "CREATE TABLE IF NOT EXISTS options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))", + "CREATE TABLE IF NOT EXISTS messages (network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)", + "CREATE INDEX IF NOT EXISTS network_channel ON messages (network, channel)", + "CREATE INDEX IF NOT EXISTS time ON messages (time)", +]; + +class MessageStorage { + constructor() { + this.isEnabled = false; + } + + enable(name) { + const logsPath = path.join(Helper.getHomePath(), "logs"); + const sqlitePath = path.join(logsPath, `${name}.sqlite3`); + + try { + fsextra.ensureDirSync(logsPath); + } catch (e) { + log.error("Unable to create logs directory", e); + + return; + } + + this.isEnabled = true; + + this.database = new sqlite3.cached.Database(sqlitePath); + this.database.serialize(() => { + schema.forEach((line) => this.database.run(line)); + + this.database.get("SELECT value FROM options WHERE name = 'schema_version'", (err, row) => { + if (err) { + return log.error(`Failed to retrieve schema version: ${err}`); + } + + // New table + if (row === undefined) { + this.database.serialize(() => this.database.run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", currentSchemaVersion)); + + return; + } + + const storedSchemaVersion = parseInt(row.value, 10); + + if (storedSchemaVersion === currentSchemaVersion) { + return; + } + + if (storedSchemaVersion > currentSchemaVersion) { + return log.error(`sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`); + } + + log.info(`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`); + + this.database.serialize(() => this.database.run("UPDATE options SET value = ? WHERE name = 'schema_version'", currentSchemaVersion)); + }); + }); + } + + index(network, channel, msg) { + if (!this.isEnabled) { + return; + } + + const clonedMsg = Object.keys(msg).reduce((newMsg, prop) => { + // id is regenerated when messages are retrieved + // previews are not stored because storage is cleared on lounge restart + // type and time are stored in a separate column + if (prop !== "id" && prop !== "previews" && prop !== "type" && prop !== "time") { + newMsg[prop] = msg[prop]; + } + + return newMsg; + }, {}); + + this.database.serialize(() => this.database.run( + "INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)", + network, channel.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg) + )); + } + + /** + * Load messages for given channel on a given network and resolve a promise with loaded messages. + * + * @param Network network - Network object where the channel is + * @param Chan channel - Channel object for which to load messages for + */ + getMessages(network, channel) { + if (!this.isEnabled || Helper.config.maxHistory < 1) { + return Promise.resolve([]); + } + + return new Promise((resolve, reject) => { + this.database.parallelize(() => this.database.all( + "SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?", + [network.uuid, channel.name.toLowerCase(), Helper.config.maxHistory], + (err, rows) => { + if (err) { + return reject(err); + } + + resolve(rows.map((row) => { + const msg = JSON.parse(row.msg); + msg.time = row.time; + msg.type = row.type; + + return new Msg(msg); + }).reverse()); + } + )); + }); + } +} + +module.exports = MessageStorage; diff --git a/test/fixtures/.gitignore b/test/fixtures/.gitignore index d23abb24..7f5712ec 100644 --- a/test/fixtures/.gitignore +++ b/test/fixtures/.gitignore @@ -1,5 +1,6 @@ # Files that may be generated by tests .thelounge/storage/ +.thelounge/logs/ # Fixtures contain fake packages, stored in a fake node_modules folder !.thelounge/packages/node_modules/ diff --git a/yarn.lock b/yarn.lock index a9207058..146a9fa2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -70,8 +70,8 @@ ajv@^5.0.0, ajv@^5.1.0, ajv@^5.2.3, ajv@^5.3.0: json-schema-traverse "^0.3.0" ajv@^6.0.1, ajv@^6.1.0: - version "6.2.1" - resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.1.tgz#28a6abc493a2abe0fb4c8507acaedb43fa550671" + version "6.2.0" + resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.0.tgz#afac295bbaa0152449e522742e4547c1ae9328d2" dependencies: fast-deep-equal "^1.0.0" fast-json-stable-stringify "^2.0.0" @@ -1199,12 +1199,12 @@ caniuse-api@^1.5.2: lodash.uniq "^4.5.0" caniuse-db@^1.0.30000529, caniuse-db@^1.0.30000634, caniuse-db@^1.0.30000639: - version "1.0.30000813" - resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000813.tgz#e0a1c603f8880ad787b2a35652b2733f32a5e29a" + version "1.0.30000811" + resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000811.tgz#19efb9238393d40078332c34485c818d641c4305" caniuse-lite@^1.0.30000792, caniuse-lite@^1.0.30000809, caniuse-lite@^1.0.30000810: - version "1.0.30000813" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000813.tgz#7b25e27fdfb8d133f3c932b01f77452140fcc6c9" + version "1.0.30000811" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000811.tgz#0b6e40f2efccc27bd3cb52f91ee7ca4673d77d10" capture-stack-trace@^1.0.0: version "1.0.0" @@ -2087,8 +2087,8 @@ ee-first@1.1.1: resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d" electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.30, electron-to-chromium@^1.3.33: - version "1.3.36" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.36.tgz#0eabf71a9ebea9013fb1cc35a390e068624f27e8" + version "1.3.34" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.34.tgz#d93498f40391bb0c16a603d8241b9951404157ed" elliptic@^6.0.0: version "6.4.0" @@ -4421,6 +4421,10 @@ nan@^2.3.0: version "2.9.2" resolved "https://registry.yarnpkg.com/nan/-/nan-2.9.2.tgz#f564d75f5f8f36a6d9456cca7a6c4fe488ab7866" +nan@~2.7.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46" + nanomatch@^1.2.9: version "1.2.9" resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.9.tgz#879f7150cb2dab7a471259066c104eee6e0fa7c2" @@ -4486,7 +4490,7 @@ node-libs-browser@^2.0.0: util "^0.10.3" vm-browserify "0.0.4" -node-pre-gyp@^0.6.39: +node-pre-gyp@^0.6.39, node-pre-gyp@~0.6.38: version "0.6.39" resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649" dependencies: @@ -6272,6 +6276,13 @@ sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" +sqlite3@3.1.13: + version "3.1.13" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9" + dependencies: + nan "~2.7.0" + node-pre-gyp "~0.6.38" + sshpk@^1.7.0: version "1.13.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3" @@ -6979,7 +6990,7 @@ utils-merge@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" -uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0: +uuid@3.2.1, uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0: version "3.2.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"