Merge pull request #1839 from thelounge/sqlite

Add sqlite logging and reloading messages

defaults/config.js

@@ -148,6 +148,19 @@ module.exports = {
 	// @default  null
 	webirc: null,
 
+	//
+	// Message logging
+	// Logging is also controlled per user individually (logs variable)
+	// Leave the array empty to disable all logging globally
+	//
+	// text: Text file per network/channel in user folder
+	// sqlite: Messages are stored in SQLite, this allows them to be reloaded on server restart
+	//
+	// @type     array
+	// @default  ["sqlite", "text"]
+	//
+	messageStorage: ["sqlite", "text"],
+
 	//
 	// Log settings
 	//

package.json

@@ -56,9 +56,11 @@
     "semver": "5.5.0",
     "socket.io": "2.0.4",
     "spdy": "3.4.7",
+    "sqlite3": "3.1.13",
     "thelounge-ldapjs-non-maintained-fork": "1.0.2",
     "ua-parser-js": "0.7.17",
     "urijs": "1.19.1",
+    "uuid": "3.2.1",
     "web-push": "3.3.0",
     "yarn": "1.5.1"
   },

src/client.js

@@ -10,6 +10,7 @@ const Network = require("./models/network");
 const ircFramework = require("irc-framework");
 const Helper = require("./helper");
 const UAParser = require("ua-parser-js");
+const MessageStorage = require("./plugins/sqlite");
 
 module.exports = Client;
 
@@ -78,8 +79,16 @@ function Client(manager, name, config = {}) {
 	});
 
 	const client = this;
-
 	let delay = 0;
+
+	if (!Helper.config.public) {
+		client.messageStorage = new MessageStorage();
+
+		if (client.config.log && Helper.config.messageStorage.includes("sqlite")) {
+			client.messageStorage.enable(client.name);
+		}
+	}
+
 	(client.config.networks || []).forEach((n) => {
 		setTimeout(function() {
 			client.connect(n);
@@ -174,6 +183,7 @@ Client.prototype.connect = function(args) {
 	args.hostname = args.hostname || (client.config && client.config.hostname) || client.hostname;
 
 	const network = new Network({
+		uuid: args.uuid,
 		name: args.name || (Helper.config.displayNetwork ? "" : Helper.config.defaults.name) || "",
 		host: args.host || "",
 		port: parseInt(args.port, 10) || (args.tls ? 6697 : 6667),
@@ -261,6 +271,7 @@ Client.prototype.connect = function(args) {
 
 	network.irc.requestCap([
 		"znc.in/self-message", // Legacy echo-message for ZNc
+		"znc.in/playback", // http://wiki.znc.in/Playback
 	]);
 
 	events.forEach((plugin) => {
@@ -273,6 +284,8 @@ Client.prototype.connect = function(args) {
 	network.irc.connect();
 
 	client.save();
+
+	channels.forEach((channel) => channel.loadMessages(client, network));
 };
 
 Client.prototype.generateToken = function(callback) {

src/models/chan.js

@@ -63,11 +63,7 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
 		return;
 	}
 
-	this.messages.push(msg);
+	this.writeUserLog(client, msg);
-
-	if (client.config.log === true) {
-		writeUserLog.call(this, client, msg);
-	}
 
 	if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
 		const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
@@ -183,21 +179,86 @@ Chan.prototype.getFilteredClone = function(lastActiveChannel, lastMessage) {
 	}, {});
 };
 
-function writeUserLog(client, msg) {
+Chan.prototype.writeUserLog = function(client, msg) {
-	if (!msg.isLoggable()) {
+	this.messages.push(msg);
-		return false;
+
+	// Does this user have logs disabled
+	if (!client.config.log) {
+		return;
 	}
 
+	// Are logs disabled server-wide
+	if (Helper.config.messageStorage.length === 0) {
+		return;
+	}
+
+	// Is this particular message or channel loggable
+	if (!msg.isLoggable() || !this.isLoggable()) {
+		return;
+	}
+
+	// Find the parent network where this channel is in
 	const target = client.find(this.id);
 
 	if (!target) {
-		return false;
+		return;
 	}
 
-	userLog.write(
+	// TODO: Something more pluggable
-		client.name,
+	if (Helper.config.messageStorage.includes("sqlite")) {
-		target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs
+		client.messageStorage.index(target.network.uuid, this.name, msg);
-		this.type === Chan.Type.LOBBY ? target.network.host : this.name,
+	}
-		msg
+
-	);
+	if (Helper.config.messageStorage.includes("text")) {
+		userLog.write(
+			client.name,
+			target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs
+			this.type === Chan.Type.LOBBY ? target.network.host : this.name,
+			msg
+		);
+	}
+};
+
+Chan.prototype.loadMessages = function(client, network) {
+	if (!client.messageStorage || !this.isLoggable()) {
+		return;
+	}
+
+	client.messageStorage
+		.getMessages(network, this)
+		.then((messages) => {
+			if (messages.length === 0) {
+				if (network.irc.network.cap.isEnabled("znc.in/playback")) {
+					requestZncPlayback(this, network, 0);
+				}
+
+				return;
+			}
+
+			this.messages.unshift(...messages);
+
+			if (!this.firstUnread) {
+				this.firstUnread = messages[messages.length - 1].id;
+			}
+
+			client.emit("more", {
+				chan: this.id,
+				messages: messages.slice(-100),
+			});
+
+			if (network.irc.network.cap.isEnabled("znc.in/playback")) {
+				const from = Math.floor(messages[messages.length - 1].time.getTime() / 1000);
+
+				requestZncPlayback(this, network, from);
+			}
+		})
+		.catch((err) => log.error(`Failed to load messages: ${err}`));
+};
+
+Chan.prototype.isLoggable = function() {
+	return this.type === Chan.Type.CHANNEL || this.type === Chan.Type.QUERY;
+};
+
+function requestZncPlayback(channel, network, from) {
+	network.irc.raw("ZNC", "*playback", "PLAY", channel.name, from.toString());
 }

src/models/msg.js

@@ -40,6 +40,7 @@ class Msg {
 
 	isLoggable() {
 		return this.type !== Msg.Type.MOTD &&
+			this.type !== Msg.Type.ERROR &&
 			this.type !== Msg.Type.BANLIST &&
 			this.type !== Msg.Type.WHOIS;
 	}

src/models/network.js

@@ -1,6 +1,7 @@
 "use strict";
 
 const _ = require("lodash");
+const uuidv4 = require("uuid/v4");
 const Chan = require("./chan");
 
 module.exports = Network;
@@ -42,6 +43,10 @@ function Network(attr) {
 		chanCache: [],
 	});
 
+	if (!this.uuid) {
+		this.uuid = uuidv4();
+	}
+
 	if (!this.name) {
 		this.name = this.host;
 	}
@@ -125,6 +130,7 @@ Network.prototype.getNetworkStatus = function() {
 
 Network.prototype.export = function() {
 	const network = _.pick(this, [
+		"uuid",
 		"awayMessage",
 		"nick",
 		"name",

src/plugins/inputs/query.js

@@ -54,4 +54,5 @@ exports.input = function(network, chan, cmd, args) {
 		shouldOpen: true,
 	});
 	this.save();
+	newChan.loadMessages(this, network);
 };

src/plugins/irc-events/join.js

@@ -22,6 +22,8 @@ module.exports = function(irc, network) {
 				chan: chan.getFilteredClone(true),
 			});
 
+			chan.loadMessages(client, network);
+
 			// Request channels' modes
 			network.irc.raw("MODE", chan.name);
 		} else if (data.nick === irc.user.nick) {

src/plugins/irc-events/message.js

@@ -72,6 +72,7 @@ module.exports = function(irc, network) {
 						network: network.id,
 						chan: chan.getFilteredClone(true),
 					});
+					chan.loadMessages(client, network);
 				}
 			}
 

src/plugins/irc-events/whois.js

@@ -19,6 +19,7 @@ module.exports = function(irc, network) {
 				network: network.id,
 				chan: chan.getFilteredClone(true),
 			});
+			chan.loadMessages(client, network);
 		}
 
 		let msg;

src/plugins/sqlite.js

@@ -0,0 +1,126 @@
+"use strict";
+
+const path = require("path");
+const fsextra = require("fs-extra");
+const sqlite3 = require("sqlite3");
+const Helper = require("../helper");
+const Msg = require("../models/msg");
+
+const currentSchemaVersion = 1520239200;
+
+const schema = [
+	// Schema version #1
+	"CREATE TABLE IF NOT EXISTS options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))",
+	"CREATE TABLE IF NOT EXISTS messages (network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)",
+	"CREATE INDEX IF NOT EXISTS network_channel ON messages (network, channel)",
+	"CREATE INDEX IF NOT EXISTS time ON messages (time)",
+];
+
+class MessageStorage {
+	constructor() {
+		this.isEnabled = false;
+	}
+
+	enable(name) {
+		const logsPath = path.join(Helper.getHomePath(), "logs");
+		const sqlitePath = path.join(logsPath, `${name}.sqlite3`);
+
+		try {
+			fsextra.ensureDirSync(logsPath);
+		} catch (e) {
+			log.error("Unable to create logs directory", e);
+
+			return;
+		}
+
+		this.isEnabled = true;
+
+		this.database = new sqlite3.cached.Database(sqlitePath);
+		this.database.serialize(() => {
+			schema.forEach((line) => this.database.run(line));
+
+			this.database.get("SELECT value FROM options WHERE name = 'schema_version'", (err, row) => {
+				if (err) {
+					return log.error(`Failed to retrieve schema version: ${err}`);
+				}
+
+				// New table
+				if (row === undefined) {
+					this.database.serialize(() => this.database.run("INSERT INTO options (name, value) VALUES ('schema_version', ?)", currentSchemaVersion));
+
+					return;
+				}
+
+				const storedSchemaVersion = parseInt(row.value, 10);
+
+				if (storedSchemaVersion === currentSchemaVersion) {
+					return;
+				}
+
+				if (storedSchemaVersion > currentSchemaVersion) {
+					return log.error(`sqlite messages schema version is higher than expected (${storedSchemaVersion} > ${currentSchemaVersion}). Is The Lounge out of date?`);
+				}
+
+				log.info(`sqlite messages schema version is out of date (${storedSchemaVersion} < ${currentSchemaVersion}). Running migrations if any.`);
+
+				this.database.serialize(() => this.database.run("UPDATE options SET value = ? WHERE name = 'schema_version'", currentSchemaVersion));
+			});
+		});
+	}
+
+	index(network, channel, msg) {
+		if (!this.isEnabled) {
+			return;
+		}
+
+		const clonedMsg = Object.keys(msg).reduce((newMsg, prop) => {
+			// id is regenerated when messages are retrieved
+			// previews are not stored because storage is cleared on lounge restart
+			// type and time are stored in a separate column
+			if (prop !== "id" && prop !== "previews" && prop !== "type" && prop !== "time") {
+				newMsg[prop] = msg[prop];
+			}
+
+			return newMsg;
+		}, {});
+
+		this.database.serialize(() => this.database.run(
+			"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)",
+			network, channel.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg)
+		));
+	}
+
+	/**
+	 * Load messages for given channel on a given network and resolve a promise with loaded messages.
+	 *
+	 * @param Network network - Network object where the channel is
+	 * @param Chan channel - Channel object for which to load messages for
+	 */
+	getMessages(network, channel) {
+		if (!this.isEnabled || Helper.config.maxHistory < 1) {
+			return Promise.resolve([]);
+		}
+
+		return new Promise((resolve, reject) => {
+			this.database.parallelize(() => this.database.all(
+				"SELECT msg, type, time FROM messages WHERE network = ? AND channel = ? ORDER BY time DESC LIMIT ?",
+				[network.uuid, channel.name.toLowerCase(), Helper.config.maxHistory],
+				(err, rows) => {
+					if (err) {
+						return reject(err);
+					}
+
+					resolve(rows.map((row) => {
+						const msg = JSON.parse(row.msg);
+						msg.time = row.time;
+						msg.type = row.type;
+
+						return new Msg(msg);
+					}).reverse());
+				}
+			));
+		});
+	}
+}
+
+module.exports = MessageStorage;

src/server.js

@@ -306,6 +306,7 @@ function initializeClient(socket, client, token, lastMessage) {
 			// prevent people from overriding webirc settings
 			data.ip = null;
 			data.hostname = null;
+			data.uuid = null;
 
 			client.connect(data);
 		}

test/fixtures/.gitignore

@@ -1,5 +1,6 @@
 # Files that may be generated by tests
 .thelounge/storage/
+.thelounge/logs/
 
 # Fixtures contain fake packages, stored in a fake node_modules folder
 !.thelounge/packages/node_modules/

test/models/network.js

@@ -10,6 +10,7 @@ describe("Network", function() {
 	describe("#export()", function() {
 		it("should produce an valid object", function() {
 			const network = new Network({
+				uuid: "hello world",
 				awayMessage: "I am away",
 				name: "networkName",
 				channels: [
@@ -24,6 +25,7 @@ describe("Network", function() {
 			network.setNick("chillin`");
 
 			expect(network.export()).to.deep.equal({
+				uuid: "hello world",
 				awayMessage: "I am away",
 				name: "networkName",
 				host: "",
@@ -47,6 +49,15 @@ describe("Network", function() {
 			});
 		});
 
+		it("should generate uuid (v4) for each network", function() {
+			const network1 = new Network();
+			const network2 = new Network();
+
+			expect(network1.uuid).to.have.lengthOf(36);
+			expect(network2.uuid).to.have.lengthOf(36);
+			expect(network1.uuid).to.not.equal(network2.uuid);
+		});
+
 		it("lobby should be at the top", function() {
 			const network = new Network({
 				name: "Super Nice Network",
@@ -126,6 +137,7 @@ describe("Network", function() {
 				"status",
 				"tls",
 				"rejectUnauthorized",
+				"uuid",
 				"username"
 			);
 

test/plugins/sqlite.js

@@ -0,0 +1,102 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const expect = require("chai").expect;
+const Msg = require("../../src/models/msg");
+const Helper = require("../../src/helper");
+const MessageStorage = require("../../src/plugins/sqlite.js");
+
+describe("SQLite Message Storage", function() {
+	const expectedPath = path.join(Helper.getHomePath(), "logs", "testUser.sqlite3");
+	let store;
+
+	// Delete database file from previous test run
+	before(function(done) {
+		store = new MessageStorage();
+
+		if (fs.existsSync(expectedPath)) {
+			fs.unlink(expectedPath, done);
+		} else {
+			done();
+		}
+	});
+
+	it("should resolve an empty array when disabled", function(done) {
+		store.getMessages(null, null).then((messages) => {
+			expect(messages).to.be.empty;
+			done();
+		});
+	});
+
+	it("should create database file", function() {
+		expect(store.isEnabled).to.be.false;
+		expect(fs.existsSync(expectedPath)).to.be.false;
+
+		store.enable("testUser");
+
+		expect(store.isEnabled).to.be.true;
+		expect(fs.existsSync(expectedPath)).to.be.true;
+	});
+
+	it("should create tables", function(done) {
+		store.database.serialize(() =>
+			store.database.all("SELECT name, tbl_name, sql FROM sqlite_master WHERE type = 'table'", (err, row) => {
+				expect(err).to.be.null;
+				expect(row).to.deep.equal([{
+					name: "options",
+					tbl_name: "options",
+					sql: "CREATE TABLE options (name TEXT, value TEXT, CONSTRAINT name_unique UNIQUE (name))",
+				},
+				{
+					name: "messages",
+					tbl_name: "messages",
+					sql: "CREATE TABLE messages (network TEXT, channel TEXT, time INTEGER, type TEXT, msg TEXT)",
+				}]);
+
+				done();
+			})
+		);
+	});
+
+	it("should insert schema version to options table", function(done) {
+		store.database.serialize(() =>
+			store.database.get("SELECT value FROM options WHERE name = 'schema_version'", (err, row) => {
+				expect(err).to.be.null;
+
+				// Should be sqlite.currentSchemaVersion,
+				// compared as string because it's returned as such from the database
+				expect(row.value).to.equal("1520239200");
+
+				done();
+			})
+		);
+	});
+
+	it("should store a message", function(done) {
+		store.index("this-is-a-network-guid", "#ThisIsAChannel", new Msg({
+			time: 123456789,
+			text: "Hello from sqlite world!",
+		}));
+
+		store.database.serialize(done);
+	});
+
+	it("should retrieve previously stored message", function(done) {
+		store.getMessages({
+			uuid: "this-is-a-network-guid",
+		}, {
+			name: "#thisisaCHANNEL",
+		}).then((messages) => {
+			expect(messages).to.have.lengthOf(1);
+
+			const msg = messages[0];
+
+			expect(msg.text).to.equal("Hello from sqlite world!");
+			expect(msg.type).to.equal(Msg.Type.MESSAGE);
+			expect(msg.time.getTime()).to.equal(123456789);
+
+			done();
+		});
+	});
+});

yarn.lock

@@ -70,8 +70,8 @@ ajv@^5.0.0, ajv@^5.1.0, ajv@^5.2.3, ajv@^5.3.0:
     json-schema-traverse "^0.3.0"
 
 ajv@^6.0.1, ajv@^6.1.0:
-  version "6.2.1"
+  version "6.2.0"
-  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.1.tgz#28a6abc493a2abe0fb4c8507acaedb43fa550671"
+  resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.2.0.tgz#afac295bbaa0152449e522742e4547c1ae9328d2"
   dependencies:
     fast-deep-equal "^1.0.0"
     fast-json-stable-stringify "^2.0.0"
@@ -1199,12 +1199,12 @@ caniuse-api@^1.5.2:
     lodash.uniq "^4.5.0"
 
 caniuse-db@^1.0.30000529, caniuse-db@^1.0.30000634, caniuse-db@^1.0.30000639:
-  version "1.0.30000813"
+  version "1.0.30000811"
-  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000813.tgz#e0a1c603f8880ad787b2a35652b2733f32a5e29a"
+  resolved "https://registry.yarnpkg.com/caniuse-db/-/caniuse-db-1.0.30000811.tgz#19efb9238393d40078332c34485c818d641c4305"
 
 caniuse-lite@^1.0.30000792, caniuse-lite@^1.0.30000809, caniuse-lite@^1.0.30000810:
-  version "1.0.30000813"
+  version "1.0.30000811"
-  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000813.tgz#7b25e27fdfb8d133f3c932b01f77452140fcc6c9"
+  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30000811.tgz#0b6e40f2efccc27bd3cb52f91ee7ca4673d77d10"
 
 capture-stack-trace@^1.0.0:
   version "1.0.0"
@@ -2087,8 +2087,8 @@ ee-first@1.1.1:
   resolved "https://registry.yarnpkg.com/ee-first/-/ee-first-1.1.1.tgz#590c61156b0ae2f4f0255732a158b266bc56b21d"
 
 electron-to-chromium@^1.2.7, electron-to-chromium@^1.3.30, electron-to-chromium@^1.3.33:
-  version "1.3.36"
+  version "1.3.34"
-  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.36.tgz#0eabf71a9ebea9013fb1cc35a390e068624f27e8"
+  resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.34.tgz#d93498f40391bb0c16a603d8241b9951404157ed"
 
 elliptic@^6.0.0:
   version "6.4.0"
@@ -4421,6 +4421,10 @@ nan@^2.3.0:
   version "2.9.2"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.9.2.tgz#f564d75f5f8f36a6d9456cca7a6c4fe488ab7866"
 
+nan@~2.7.0:
+  version "2.7.0"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.7.0.tgz#d95bf721ec877e08db276ed3fc6eb78f9083ad46"
+
 nanomatch@^1.2.9:
   version "1.2.9"
   resolved "https://registry.yarnpkg.com/nanomatch/-/nanomatch-1.2.9.tgz#879f7150cb2dab7a471259066c104eee6e0fa7c2"
@@ -4486,7 +4490,7 @@ node-libs-browser@^2.0.0:
     util "^0.10.3"
     vm-browserify "0.0.4"
 
-node-pre-gyp@^0.6.39:
+node-pre-gyp@^0.6.39, node-pre-gyp@~0.6.38:
   version "0.6.39"
   resolved "https://registry.yarnpkg.com/node-pre-gyp/-/node-pre-gyp-0.6.39.tgz#c00e96860b23c0e1420ac7befc5044e1d78d8649"
   dependencies:
@@ -6272,6 +6276,13 @@ sprintf-js@~1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
 
+sqlite3@3.1.13:
+  version "3.1.13"
+  resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-3.1.13.tgz#d990a05627392768de6278bafd1a31fdfe907dd9"
+  dependencies:
+    nan "~2.7.0"
+    node-pre-gyp "~0.6.38"
+
 sshpk@^1.7.0:
   version "1.13.1"
   resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.13.1.tgz#512df6da6287144316dc4c18fe1cf1d940739be3"
@@ -6979,7 +6990,7 @@ utils-merge@1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713"
 
-uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
+uuid@3.2.1, uuid@^3.0.0, uuid@^3.0.1, uuid@^3.1.0:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.2.1.tgz#12c528bb9d58d0b9265d9a2f6f0fe8be17ff1f14"