Merge pull request #1307 from thelounge/xpaw/image-proxy

Store preview images on disk for privacy, security and caching

.gitignore

@@ -4,6 +4,7 @@ package-lock.json
 
 .nyc_output/
 coverage/
+test/fixtures/.lounge/storage/
 
 # Built assets created at npm install/prepublish time
 # See https://docs.npmjs.com/misc/scripts

client/views/msg_preview.tpl

@@ -2,7 +2,7 @@
 <div class="toggle-content toggle-type-{{type}}{{#if shown}} show{{/if}}">
 	{{#equal type "image"}}
 		<a class="toggle-thumbnail" href="{{link}}" target="_blank" rel="noopener">
-			<img src="{{link}}">
+			<img src="{{thumb}}">
 		</a>
 	{{else}}
 		{{#if thumb}}

defaults/config.js

@@ -66,6 +66,23 @@ module.exports = {
 	//
 	prefetch: false,
 
+	//
+	// Store and proxy prefetched images and thumbnails.
+	// This improves security and privacy by not exposing client IP address,
+	// and always loading images from The Lounge instance and making all assets secure,
+	// which in result fixes mixed content warnings.
+	//
+	// If storage is enabled, The Lounge will fetch and store images and thumbnails
+	// in ~/.lounge/storage folder, or %HOME%/storage if --home is used.
+	//
+	// Images are deleted when they are no longer referenced by any message (controlled by maxHistory),
+	// and the folder is cleaned up on every The Lounge restart.
+	//
+	// @type     boolean
+	// @default  false
+	//
+	prefetchStorage: false,
+
 	//
 	// Prefetch URLs Image Preview size limit
 	//

src/helper.js

@@ -12,6 +12,7 @@ const colors = require("colors/safe");
 var Helper = {
 	config: null,
 	expandHome: expandHome,
+	getStoragePath: getStoragePath,
 	getUserConfigPath: getUserConfigPath,
 	getUserLogsPath: getUserLogsPath,
 	setHome: setHome,
@@ -90,6 +91,10 @@ function getUserLogsPath(name, network) {
 	return path.join(this.HOME, "logs", name, network);
 }
 
+function getStoragePath() {
+	return path.join(this.HOME, "storage");
+}
+
 function ip2hex(address) {
 	// no ipv6 support
 	if (!net.isIPv4(address)) {

src/models/chan.js

@@ -2,6 +2,7 @@
 
 var _ = require("lodash");
 var Helper = require("../helper");
+const storage = require("../plugins/storage");
 
 module.exports = Chan;
 
@@ -53,7 +54,15 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
 	this.messages.push(msg);
 
 	if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
-		this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
+		const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
+
+		if (Helper.config.prefetch && Helper.config.prefetchStorage) {
+			deleted.forEach((deletedMessage) => {
+				if (deletedMessage.preview && deletedMessage.preview.thumb) {
+					storage.dereference(deletedMessage.preview.thumb);
+				}
+			});
+		}
 	}
 
 	if (!msg.self && !isOpen) {

src/plugins/irc-events/link.js

@@ -5,6 +5,7 @@ const request = require("request");
 const Helper = require("../../helper");
 const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
 const es = require("event-stream");
+const storage = require("../storage");
 
 process.setMaxListeners(0);
 
@@ -49,7 +50,7 @@ function parse(msg, url, res, client) {
 
 	switch (res.type) {
 	case "text/html":
-		var $ = cheerio.load(res.text);
+		var $ = cheerio.load(res.data);
 		preview.type = "link";
 		preview.head =
 			$("meta[property=\"og:title\"]").attr("content")
@@ -78,7 +79,7 @@ function parse(msg, url, res, client) {
 					preview.thumb = "";
 				}
 
-				emitPreview(client, msg, preview);
+				handlePreview(client, msg, preview, resThumb);
 			});
 
 			return;
@@ -90,18 +91,32 @@ function parse(msg, url, res, client) {
 	case "image/gif":
 	case "image/jpg":
 	case "image/jpeg":
-		if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) {
+		if (res.size > (Helper.config.prefetchMaxImageSize * 1024)) {
-			preview.type = "image";
-		} else {
 			return;
 		}
+
+		preview.type = "image";
+		preview.thumb = preview.link;
+
 		break;
 
 	default:
 		return;
 	}
 
-	emitPreview(client, msg, preview);
+	handlePreview(client, msg, preview, res);
+}
+
+function handlePreview(client, msg, preview, res) {
+	if (!preview.thumb.length || !Helper.config.prefetchStorage) {
+		return emitPreview(client, msg, preview);
+	}
+
+	storage.store(res.data, res.type.replace("image/", ""), (url) => {
+		preview.thumb = url;
+
+		emitPreview(client, msg, preview);
+	});
 }
 
 function emitPreview(client, msg, preview) {
@@ -164,23 +179,23 @@ function fetch(url, cb) {
 				return cb(null);
 			}
 
-			let type;
+			let type = "";
 			let size = parseInt(req.response.headers["content-length"], 10) || length;
 
 			if (size < length) {
 				size = length;
 			}
 
-			try {
+			if (req.response.headers["content-type"]) {
 				type = req.response.headers["content-type"].split(/ *; */).shift();
-			} catch (e) {
-				type = {};
 			}
+
 			data = {
-				text: data,
+				data: data,
 				type: type,
 				size: size
 			};
+
 			cb(data);
 		}));
 }

src/plugins/storage.js

@@ -0,0 +1,81 @@
+"use strict";
+
+const fs = require("fs");
+const fsextra = require("fs-extra");
+const path = require("path");
+const crypto = require("crypto");
+const helper = require("../helper");
+
+class Storage {
+	constructor() {
+		this.references = new Map();
+
+		// Ensures that a directory is empty.
+		// Deletes directory contents if the directory is not empty.
+		// If the directory does not exist, it is created.
+		fsextra.emptyDirSync(helper.getStoragePath());
+	}
+
+	dereference(url) {
+		// If maxHistory is 0, image would be dereferenced before client had a chance to retrieve it,
+		// so for now, just don't implement dereferencing for this edge case.
+		if (helper.maxHistory === 0) {
+			return;
+		}
+
+		const references = (this.references.get(url) || 0) - 1;
+
+		if (references < 0) {
+			return log.warn("Tried to dereference a file that has no references", url);
+		}
+
+		if (references > 0) {
+			return this.references.set(url, references);
+		}
+
+		this.references.delete(url);
+
+		// Drop "storage/" from url and join it with full storage path
+		const filePath = path.join(helper.getStoragePath(), url.substring(8));
+
+		fs.unlink(filePath, (err) => {
+			if (err) {
+				log.error("Failed to delete stored file", err);
+			}
+		});
+	}
+
+	store(data, extension, callback) {
+		const hash = crypto.createHash("sha256").update(data).digest("hex");
+		const a = hash.substring(0, 2);
+		const b = hash.substring(2, 4);
+		const folder = path.join(helper.getStoragePath(), a, b);
+		const filePath = path.join(folder, `${hash.substring(4)}.${extension}`);
+		const url = `storage/${a}/${b}/${hash.substring(4)}.${extension}`;
+
+		this.references.set(url, 1 + (this.references.get(url) || 0));
+
+		// If file with this name already exists, we don't need to write it again
+		if (fs.existsSync(filePath)) {
+			return callback(url);
+		}
+
+		fsextra.ensureDir(folder).then(() => {
+			fs.writeFile(filePath, data, (err) => {
+				if (err) {
+					log.error("Failed to store a file", err);
+
+					return callback("");
+				}
+
+				callback(url);
+			});
+		}).catch((err) => {
+			log.error("Failed to create storage folder", err);
+
+			return callback("");
+		});
+	}
+}
+
+module.exports = new Storage();

src/server.js

@@ -33,6 +33,10 @@ module.exports = function() {
 		.use(allRequests)
 		.use(index)
 		.use(express.static("client"))
+		.use("/storage/", express.static(Helper.getStoragePath(), {
+			redirect: false,
+			maxAge: 86400 * 1000,
+		}))
 		.engine("html", expressHandlebars({
 			extname: ".html",
 			helpers: {
@@ -152,7 +156,24 @@ function index(req, res, next) {
 			filename: filename
 		};
 	});
-	res.setHeader("Content-Security-Policy", "default-src *; connect-src 'self' ws: wss:; style-src * 'unsafe-inline'; script-src 'self'; child-src 'self'; object-src 'none'; form-action 'none';");
+
+	const policies = [
+		"default-src *",
+		"connect-src 'self' ws: wss:",
+		"style-src * 'unsafe-inline'",
+		"script-src 'self'",
+		"child-src 'self'",
+		"object-src 'none'",
+		"form-action 'none'",
+	];
+
+	// If prefetch is enabled, but storage is not, we have to allow mixed content
+	if (Helper.config.prefetchStorage || !Helper.config.prefetch) {
+		policies.push("img-src 'self'");
+		policies.unshift("block-all-mixed-content");
+	}
+
+	res.setHeader("Content-Security-Policy", policies.join("; "));
 	res.setHeader("Referrer-Policy", "no-referrer");
 	res.render("index", data);
 }

test/plugins/link.js

@@ -1,10 +1,10 @@
 "use strict";
 
-const expect = require("chai").expect;
-
-var util = require("../util");
-var link = require("../../src/plugins/irc-events/link.js");
 const path = require("path");
+const expect = require("chai").expect;
+const util = require("../util");
+const Helper = require("../../src/helper");
+const link = require("../../src/plugins/irc-events/link.js");
 
 describe("Link plugin", function() {
 	before(function(done) {
@@ -22,6 +22,8 @@ describe("Link plugin", function() {
 	beforeEach(function() {
 		this.irc = util.createClient();
 		this.network = util.createNetwork();
+
+		Helper.config.prefetchStorage = false;
 	});
 
 	it("should be able to fetch basic information about URLs", function(done) {
@@ -39,6 +41,7 @@ describe("Link plugin", function() {
 			expect(data.preview.type).to.equal("link");
 			expect(data.preview.head).to.equal("test title");
 			expect(data.preview.body).to.equal("simple description");
+			expect(data.preview.link).to.equal("http://localhost:9002/basic");
 			expect(message.previews.length).to.equal(1);
 			done();
 		});
@@ -104,11 +107,13 @@ describe("Link plugin", function() {
 		link(this.irc, this.network.channels[0], message);
 
 		this.app.get("/invalid-thumb", function(req, res) {
-			res.send("<title>test</title><meta property='og:image' content='/real-test-image.png'>");
+			res.send("<title>test invalid image</title><meta property='og:image' content='/real-test-image.png'>");
 		});
 
 		this.irc.once("msg:preview", function(data) {
 			expect(data.preview.thumb).to.be.empty;
+			expect(data.preview.head).to.equal("test invalid image");
+			expect(data.preview.link).to.equal("http://localhost:9002/invalid-thumb");
 			done();
 		});
 	});
@@ -127,6 +132,7 @@ describe("Link plugin", function() {
 		this.irc.once("msg:preview", function(data) {
 			expect(data.preview.head).to.equal("Untitled page");
 			expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
+			expect(data.preview.link).to.equal("http://localhost:9002/thumb-no-title");
 			done();
 		});
 	});
@@ -144,6 +150,7 @@ describe("Link plugin", function() {
 
 		this.irc.once("msg:preview", function(data) {
 			expect(data.preview.head).to.equal("404 image");
+			expect(data.preview.link).to.equal("http://localhost:9002/thumb-404");
 			expect(data.preview.thumb).to.be.empty;
 			done();
 		});
@@ -159,6 +166,7 @@ describe("Link plugin", function() {
 		this.irc.once("msg:preview", function(data) {
 			expect(data.preview.type).to.equal("image");
 			expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png");
+			expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
 			done();
 		});
 	});

test/plugins/storage.js

@@ -0,0 +1,68 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const expect = require("chai").expect;
+const util = require("../util");
+const Helper = require("../../src/helper");
+const link = require("../../src/plugins/irc-events/link.js");
+
+describe("Image storage", function() {
+	const testImagePath = path.resolve(__dirname, "../../client/img/apple-touch-icon-120x120.png");
+	const correctImageHash = crypto.createHash("sha256").update(fs.readFileSync(testImagePath)).digest("hex");
+	const correctImageURL = `storage/${correctImageHash.substring(0, 2)}/${correctImageHash.substring(2, 4)}/${correctImageHash.substring(4)}.png`;
+
+	before(function(done) {
+		this.app = util.createWebserver();
+		this.app.get("/real-test-image.png", function(req, res) {
+			res.sendFile(testImagePath);
+		});
+		this.connection = this.app.listen(9003, done);
+	});
+
+	after(function(done) {
+		this.connection.close(done);
+	});
+
+	beforeEach(function() {
+		this.irc = util.createClient();
+		this.network = util.createNetwork();
+
+		Helper.config.prefetchStorage = true;
+	});
+
+	it("should store the thumbnail", function(done) {
+		const message = this.irc.createMessage({
+			text: "http://localhost:9003/thumb"
+		});
+
+		link(this.irc, this.network.channels[0], message);
+
+		this.app.get("/thumb", function(req, res) {
+			res.send("<title>Google</title><meta property='og:image' content='http://localhost:9003/real-test-image.png'>");
+		});
+
+		this.irc.once("msg:preview", function(data) {
+			expect(data.preview.head).to.equal("Google");
+			expect(data.preview.link).to.equal("http://localhost:9003/thumb");
+			expect(data.preview.thumb).to.equal(correctImageURL);
+			done();
+		});
+	});
+
+	it("should store the image", function(done) {
+		const message = this.irc.createMessage({
+			text: "http://localhost:9003/real-test-image.png"
+		});
+
+		link(this.irc, this.network.channels[0], message);
+
+		this.irc.once("msg:preview", function(data) {
+			expect(data.preview.type).to.equal("image");
+			expect(data.preview.link).to.equal("http://localhost:9003/real-test-image.png");
+			expect(data.preview.thumb).to.equal(correctImageURL);
+			done();
+		});
+	});
+});