Merge pull request #2366 from thelounge/xpaw/refactor-userlog

Refactor userLog to be the same as sqlite logger

package.json

@@ -44,6 +44,7 @@
     "cheerio": "0.22.0",
     "commander": "2.15.1",
     "express": "4.16.3",
+    "filenamify": "2.0.0",
     "fs-extra": "6.0.1",
     "irc-framework": "2.11.0",
     "linkify-it": "2.0.3",

src/client.js

@@ -8,9 +8,11 @@ const Msg = require("./models/msg");
 const Network = require("./models/network");
 const Helper = require("./helper");
 const UAParser = require("ua-parser-js");
-const MessageStorage = require("./plugins/sqlite");
 const uuidv4 = require("uuid/v4");
 
+const MessageStorage = require("./plugins/messageStorage/sqlite");
+const TextFileMessageStorage = require("./plugins/messageStorage/text");
+
 module.exports = Client;
 
 const events = [
@@ -76,16 +78,23 @@ function Client(manager, name, config = {}) {
 		networks: [],
 		sockets: manager.sockets,
 		manager: manager,
+		messageStorage: [],
 	});
 
 	const client = this;
 	let delay = 0;
 
-	if (!Helper.config.public) {
+	if (!Helper.config.public && client.config.log) {
-		client.messageStorage = new MessageStorage(client);
+		if (Helper.config.messageStorage.includes("sqlite")) {
+			client.messageStorage.push(new MessageStorage(client));
+		}
 
-		if (client.config.log && Helper.config.messageStorage.includes("sqlite")) {
+		if (Helper.config.messageStorage.includes("text")) {
-			client.messageStorage.enable(client.name);
+			client.messageStorage.push(new TextFileMessageStorage(client));
+		}
+
+		for (const messageStorage of client.messageStorage) {
+			messageStorage.enable();
 		}
 	}
 
@@ -489,8 +498,8 @@ Client.prototype.quit = function(signOut) {
 		network.destroy();
 	});
 
-	if (this.messageStorage) {
+	for (const messageStorage of this.messageStorage) {
-		this.messageStorage.close();
+		messageStorage.close();
 	}
 };
 

src/helper.js

@@ -14,6 +14,7 @@ let configPath;
 let usersPath;
 let storagePath;
 let packagesPath;
+let userLogsPath;
 
 const Helper = {
 	config: null,
@@ -89,6 +90,7 @@ function setHome(newPath) {
 	usersPath = path.join(homePath, "users");
 	storagePath = path.join(homePath, "storage");
 	packagesPath = path.join(homePath, "packages");
+	userLogsPath = path.join(homePath, "logs");
 
 	// Reload config from new home location
 	if (fs.existsSync(configPath)) {
@@ -145,8 +147,8 @@ function getUserConfigPath(name) {
 	return path.join(usersPath, name + ".json");
 }
 
-function getUserLogsPath(name, network) {
+function getUserLogsPath() {
-	return path.join(homePath, "logs", name, network);
+	return userLogsPath;
 }
 
 function getStoragePath() {

src/models/chan.js

@@ -3,7 +3,6 @@
 const _ = require("lodash");
 const Helper = require("../helper");
 const User = require("./user");
-const userLog = require("../userLog");
 const storage = require("../plugins/storage");
 
 module.exports = Chan;
@@ -180,13 +179,8 @@ Chan.prototype.getFilteredClone = function(lastActiveChannel, lastMessage) {
 Chan.prototype.writeUserLog = function(client, msg) {
 	this.messages.push(msg);
 
-	// Does this user have logs disabled
+	// Are there any logs enabled
-	if (!client.config.log) {
+	if (client.messageStorage.length === 0) {
-		return;
-	}
-
-	// Are logs disabled server-wide
-	if (Helper.config.messageStorage.length === 0) {
 		return;
 	}
 
@@ -202,27 +196,23 @@ Chan.prototype.writeUserLog = function(client, msg) {
 		return;
 	}
 
-	// TODO: Something more pluggable
+	for (const messageStorage of client.messageStorage) {
-	if (Helper.config.messageStorage.includes("sqlite")) {
+		messageStorage.index(target.network, this, msg);
-		client.messageStorage.index(target.network.uuid, this.name, msg);
-	}
-
-	if (Helper.config.messageStorage.includes("text")) {
-		userLog.write(
-			client.name,
-			target.network.host, // TODO: Fix #1392, multiple connections to same server results in duplicate logs
-			this.type === Chan.Type.LOBBY ? target.network.host : this.name,
-			msg
-		);
 	}
 };
 
 Chan.prototype.loadMessages = function(client, network) {
-	if (!client.messageStorage || !this.isLoggable()) {
+	if (!this.isLoggable()) {
 		return;
 	}
 
-	client.messageStorage
+	const messageStorage = client.messageStorage.find((s) => s.canProvideMessages());
+
+	if (!messageStorage) {
+		return;
+	}
+
+	messageStorage
 		.getMessages(network, this)
 		.then((messages) => {
 			if (messages.length === 0) {

src/models/network.js

@@ -129,7 +129,7 @@ Network.prototype.createIrcFramework = function(client) {
 
 	// Request only new messages from ZNC if we have sqlite logging enabled
 	// See http://wiki.znc.in/Playback
-	if (client.config.log && Helper.config.messageStorage.includes("sqlite")) {
+	if (client.config.log && client.messageStorage.find((s) => s.canProvideMessages())) {
 		this.irc.requestCap("znc.in/playback");
 	}
 };

src/plugins/messageStorage/sqlite.js

@@ -2,8 +2,8 @@
 
 const path = require("path");
 const fsextra = require("fs-extra");
-const Helper = require("../helper");
+const Helper = require("../../helper");
-const Msg = require("../models/msg");
+const Msg = require("../../models/msg");
 
 let sqlite3;
 
@@ -31,9 +31,9 @@ class MessageStorage {
 		this.isEnabled = false;
 	}
 
-	enable(name) {
+	enable() {
-		const logsPath = path.join(Helper.getHomePath(), "logs");
+		const logsPath = Helper.getUserLogsPath();
-		const sqlitePath = path.join(logsPath, `${name}.sqlite3`);
+		const sqlitePath = path.join(logsPath, `${this.client.name}.sqlite3`);
 
 		try {
 			fsextra.ensureDirSync(logsPath);
@@ -114,7 +114,7 @@ class MessageStorage {
 
 		this.database.serialize(() => this.database.run(
 			"INSERT INTO messages(network, channel, time, type, msg) VALUES(?, ?, ?, ?, ?)",
-			network, channel.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg)
+			network.uuid, channel.name.toLowerCase(), msg.time.getTime(), msg.type, JSON.stringify(clonedMsg)
 		));
 	}
 
@@ -152,6 +152,10 @@ class MessageStorage {
 			));
 		});
 	}
+
+	canProvideMessages() {
+		return this.isEnabled;
+	}
 }
 
 module.exports = MessageStorage;

src/plugins/messageStorage/text.js

@@ -0,0 +1,99 @@
+"use strict";
+
+const fs = require("fs");
+const fsextra = require("fs-extra");
+const path = require("path");
+const moment = require("moment");
+const filenamify = require("filenamify");
+const Helper = require("../../helper");
+
+class TextFileMessageStorage {
+	constructor(client) {
+		this.client = client;
+		this.isEnabled = false;
+	}
+
+	enable() {
+		this.isEnabled = true;
+	}
+
+	close(callback) {
+		this.isEnabled = false;
+
+		if (callback) {
+			callback();
+		}
+	}
+
+	index(network, channel, msg) {
+		if (!this.isEnabled) {
+			return;
+		}
+
+		const networkFolderName = cleanFilename(`${network.name}-${network.uuid.substring(network.name.length + 1)}`);
+		const logPath = path.join(Helper.getUserLogsPath(), this.client.name, networkFolderName);
+
+		try {
+			fsextra.ensureDirSync(logPath);
+		} catch (e) {
+			log.error("Unable to create logs directory", e);
+			return;
+		}
+
+		const format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss";
+		const tz = Helper.config.logs.timezone || "UTC+00:00";
+
+		const time = moment(msg.time).utcOffset(tz).format(format);
+		let line = `[${time}] `;
+
+		if (msg.type === "message") {
+			// Format:
+			// [2014-01-01 00:00:00] <Arnold> Put that cookie down.. Now!!
+			line += `<${msg.from.nick}> ${msg.text}`;
+		} else {
+			// Format:
+			// [2014-01-01 00:00:00] * Arnold quit
+			line += `* ${msg.from.nick} `;
+
+			if (msg.hostmask) {
+				line += `(${msg.hostmask}) `;
+			}
+
+			line += msg.type;
+
+			if (msg.new_nick) { // `/nick <new_nick>`
+				line += ` ${msg.new_nick}`;
+			} else if (msg.text) {
+				line += ` ${msg.text}`;
+			}
+		}
+
+		line += "\n";
+
+		fs.appendFile(path.join(logPath, cleanFilename(channel.name)), line, (e) => {
+			if (e) {
+				log.error("Failed to write user log", e);
+			}
+		});
+	}
+
+	getMessages() {
+		// Not implemented for text log files
+		// They do not contain enough data to fully re-create message objects
+		// Use sqlite storage instead
+		return Promise.resolve([]);
+	}
+
+	canProvideMessages() {
+		return false;
+	}
+}
+
+module.exports = TextFileMessageStorage;
+
+function cleanFilename(name) {
+	name = filenamify(name, {replacement: "_"});
+	name = name.toLowerCase();
+
+	return `${name}.log`;
+}

src/userLog.js

@@ -1,59 +0,0 @@
-"use strict";
-
-const fs = require("fs");
-const fsextra = require("fs-extra");
-const moment = require("moment");
-const Helper = require("./helper");
-
-module.exports.write = function(user, network, chan, msg) {
-	const path = Helper.getUserLogsPath(user, network);
-
-	try {
-		fsextra.ensureDirSync(path);
-	} catch (e) {
-		log.error("Unable to create logs directory", e);
-		return;
-	}
-
-	const format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss";
-	const tz = Helper.config.logs.timezone || "UTC+00:00";
-
-	const time = moment(msg.time).utcOffset(tz).format(format);
-	let line = `[${time}] `;
-
-	const type = msg.type.trim();
-
-	if (type === "message" || type === "highlight") {
-		// Format:
-		// [2014-01-01 00:00:00] <Arnold> Put that cookie down.. Now!!
-		line += `<${msg.from.nick}> ${msg.text}`;
-	} else {
-		// Format:
-		// [2014-01-01 00:00:00] * Arnold quit
-		line += `* ${msg.from.nick} `;
-
-		if (msg.hostmask) {
-			line += `(${msg.hostmask}) `;
-		}
-
-		line += msg.type;
-
-		if (msg.new_nick) { // `/nick <new_nick>`
-			line += ` ${msg.new_nick}`;
-		} else if (msg.text) {
-			line += ` ${msg.text}`;
-		}
-	}
-
-	fs.appendFile(
-		// Quick fix to escape pre-escape channel names that contain % using %%,
-		// and / using %. **This does not escape all reserved words**
-		path + "/" + chan.replace(/%/g, "%%").replace(/\//g, "%") + ".log",
-		line + "\n",
-		function(e) {
-			if (e) {
-				log.error("Failed to write user log", e);
-			}
-		}
-	);
-};

test/plugins/sqlite.js

@@ -5,7 +5,7 @@ const path = require("path");
 const expect = require("chai").expect;
 const Msg = require("../../src/models/msg");
 const Helper = require("../../src/helper");
-const MessageStorage = require("../../src/plugins/sqlite.js");
+const MessageStorage = require("../../src/plugins/messageStorage/sqlite.js");
 
 describe("SQLite Message Storage", function() {
 	const expectedPath = path.join(Helper.getHomePath(), "logs", "testUser.sqlite3");
@@ -14,6 +14,7 @@ describe("SQLite Message Storage", function() {
 	// Delete database file from previous test run
 	before(function(done) {
 		store = new MessageStorage({
+			name: "testUser",
 			idMsg: 1,
 		});
 
@@ -35,7 +36,7 @@ describe("SQLite Message Storage", function() {
 		expect(store.isEnabled).to.be.false;
 		expect(fs.existsSync(expectedPath)).to.be.false;
 
-		store.enable("testUser");
+		store.enable();
 
 		expect(store.isEnabled).to.be.true;
 	});
@@ -76,7 +77,11 @@ describe("SQLite Message Storage", function() {
 
 	it("should store a message", function(done) {
 		store.database.serialize(() => {
-			store.index("this-is-a-network-guid", "#ThisIsAChannel", new Msg({
+			store.index({
+				uuid: "this-is-a-network-guid",
+			}, {
+				name: "#thisISaCHANNEL",
+			}, new Msg({
 				time: 123456789,
 				text: "Hello from sqlite world!",
 			}));

yarn.lock

@@ -2959,6 +2959,18 @@ filename-regex@^2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/filename-regex/-/filename-regex-2.0.1.tgz#c1c4b9bee3e09725ddb106b75c1e301fe2f18b26"
 
+filename-reserved-regex@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/filename-reserved-regex/-/filename-reserved-regex-2.0.0.tgz#abf73dfab735d045440abfea2d91f389ebbfa229"
+
+filenamify@2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/filenamify/-/filenamify-2.0.0.tgz#bd162262c0b6e94bfbcdcf19a3bbb3764f785695"
+  dependencies:
+    filename-reserved-regex "^2.0.0"
+    strip-outer "^1.0.0"
+    trim-repeated "^1.0.0"
+
 fill-range@^2.1.0:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-2.2.3.tgz#50b77dfd7e469bc7492470963699fe7a8485a723"
@@ -7375,6 +7387,12 @@ strip-json-comments@~2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a"
 
+strip-outer@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/strip-outer/-/strip-outer-1.0.1.tgz#b2fd2abf6604b9d1e6013057195df836b8a9d631"
+  dependencies:
+    escape-string-regexp "^1.0.2"
+
 style-loader@^0.19.1:
   version "0.19.1"
   resolved "https://registry.yarnpkg.com/style-loader/-/style-loader-0.19.1.tgz#591ffc80bcefe268b77c5d9ebc0505d772619f85"
@@ -7670,6 +7688,12 @@ trim-newlines@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/trim-newlines/-/trim-newlines-2.0.0.tgz#b403d0b91be50c331dfc4b82eeceb22c3de16d20"
 
+trim-repeated@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/trim-repeated/-/trim-repeated-1.0.0.tgz#e3646a2ea4e891312bf7eace6cfb05380bc01c21"
+  dependencies:
+    escape-string-regexp "^1.0.2"
+
 trim-right@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/trim-right/-/trim-right-1.0.1.tgz#cb2e1203067e0c8de1f614094b9fe45704ea6003"