From f35a2809a7a5d022a1a1fcba2548b85877395cc9 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Thu, 6 Jul 2017 18:33:09 +0300 Subject: [PATCH] Store preview images on disk for privacy, security and caching --- .gitignore | 1 + client/views/msg_preview.tpl | 2 +- defaults/config.js | 17 +++++++ src/helper.js | 5 +++ src/models/chan.js | 11 ++++- src/plugins/irc-events/link.js | 37 +++++++++++----- src/plugins/storage.js | 81 ++++++++++++++++++++++++++++++++++ src/server.js | 23 +++++++++- test/plugins/link.js | 18 +++++--- test/plugins/storage.js | 68 ++++++++++++++++++++++++++++ 10 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 src/plugins/storage.js create mode 100644 test/plugins/storage.js diff --git a/.gitignore b/.gitignore index 3c7c410b..58e6880a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ package-lock.json .nyc_output/ coverage/ +test/fixtures/.lounge/storage/ # Built assets created at npm install/prepublish time # See https://docs.npmjs.com/misc/scripts diff --git a/client/views/msg_preview.tpl b/client/views/msg_preview.tpl index 24f4a599..aa20c363 100644 --- a/client/views/msg_preview.tpl +++ b/client/views/msg_preview.tpl @@ -2,7 +2,7 @@
{{#equal type "image"}} - + {{else}} {{#if thumb}} diff --git a/defaults/config.js b/defaults/config.js index 83d4c7e8..9f757e5f 100644 --- a/defaults/config.js +++ b/defaults/config.js @@ -66,6 +66,23 @@ module.exports = { // prefetch: false, + // + // Store and proxy prefetched images and thumbnails. + // This improves security and privacy by not exposing client IP address, + // and always loading images from The Lounge instance and making all assets secure, + // which in result fixes mixed content warnings. + // + // If storage is enabled, The Lounge will fetch and store images and thumbnails + // in ~/.lounge/storage folder, or %HOME%/storage if --home is used. + // + // Images are deleted when they are no longer referenced by any message (controlled by maxHistory), + // and the folder is cleaned up on every The Lounge restart. + // + // @type boolean + // @default false + // + prefetchStorage: false, + // // Prefetch URLs Image Preview size limit // diff --git a/src/helper.js b/src/helper.js index 23c4abfb..2fae848e 100644 --- a/src/helper.js +++ b/src/helper.js @@ -12,6 +12,7 @@ const colors = require("colors/safe"); var Helper = { config: null, expandHome: expandHome, + getStoragePath: getStoragePath, getUserConfigPath: getUserConfigPath, getUserLogsPath: getUserLogsPath, setHome: setHome, @@ -90,6 +91,10 @@ function getUserLogsPath(name, network) { return path.join(this.HOME, "logs", name, network); } +function getStoragePath() { + return path.join(this.HOME, "storage"); +} + function ip2hex(address) { // no ipv6 support if (!net.isIPv4(address)) { diff --git a/src/models/chan.js b/src/models/chan.js index 3ac014ae..66418195 100644 --- a/src/models/chan.js +++ b/src/models/chan.js @@ -2,6 +2,7 @@ var _ = require("lodash"); var Helper = require("../helper"); +const storage = require("../plugins/storage"); module.exports = Chan; @@ -53,7 +54,15 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) { this.messages.push(msg); if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) { - this.messages.splice(0, this.messages.length - Helper.config.maxHistory); + const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory); + + if (Helper.config.prefetch && Helper.config.prefetchStorage) { + deleted.forEach((deletedMessage) => { + if (deletedMessage.preview && deletedMessage.preview.thumb) { + storage.dereference(deletedMessage.preview.thumb); + } + }); + } } if (!msg.self && !isOpen) { diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js index e0f0120f..fbf00479 100644 --- a/src/plugins/irc-events/link.js +++ b/src/plugins/irc-events/link.js @@ -5,6 +5,7 @@ const request = require("request"); const Helper = require("../../helper"); const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks"); const es = require("event-stream"); +const storage = require("../storage"); process.setMaxListeners(0); @@ -49,7 +50,7 @@ function parse(msg, url, res, client) { switch (res.type) { case "text/html": - var $ = cheerio.load(res.text); + var $ = cheerio.load(res.data); preview.type = "link"; preview.head = $("meta[property=\"og:title\"]").attr("content") @@ -78,7 +79,7 @@ function parse(msg, url, res, client) { preview.thumb = ""; } - emitPreview(client, msg, preview); + handlePreview(client, msg, preview, resThumb); }); return; @@ -90,18 +91,32 @@ function parse(msg, url, res, client) { case "image/gif": case "image/jpg": case "image/jpeg": - if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) { - preview.type = "image"; - } else { + if (res.size > (Helper.config.prefetchMaxImageSize * 1024)) { return; } + + preview.type = "image"; + preview.thumb = preview.link; + break; default: return; } - emitPreview(client, msg, preview); + handlePreview(client, msg, preview, res); +} + +function handlePreview(client, msg, preview, res) { + if (!preview.thumb.length || !Helper.config.prefetchStorage) { + return emitPreview(client, msg, preview); + } + + storage.store(res.data, res.type.replace("image/", ""), (url) => { + preview.thumb = url; + + emitPreview(client, msg, preview); + }); } function emitPreview(client, msg, preview) { @@ -164,23 +179,23 @@ function fetch(url, cb) { return cb(null); } - let type; + let type = ""; let size = parseInt(req.response.headers["content-length"], 10) || length; if (size < length) { size = length; } - try { + if (req.response.headers["content-type"]) { type = req.response.headers["content-type"].split(/ *; */).shift(); - } catch (e) { - type = {}; } + data = { - text: data, + data: data, type: type, size: size }; + cb(data); })); } diff --git a/src/plugins/storage.js b/src/plugins/storage.js new file mode 100644 index 00000000..2d889f1b --- /dev/null +++ b/src/plugins/storage.js @@ -0,0 +1,81 @@ +"use strict"; + +const fs = require("fs"); +const fsextra = require("fs-extra"); +const path = require("path"); +const crypto = require("crypto"); +const helper = require("../helper"); + +class Storage { + constructor() { + this.references = new Map(); + + // Ensures that a directory is empty. + // Deletes directory contents if the directory is not empty. + // If the directory does not exist, it is created. + fsextra.emptyDirSync(helper.getStoragePath()); + } + + dereference(url) { + // If maxHistory is 0, image would be dereferenced before client had a chance to retrieve it, + // so for now, just don't implement dereferencing for this edge case. + if (helper.maxHistory === 0) { + return; + } + + const references = (this.references.get(url) || 0) - 1; + + if (references < 0) { + return log.warn("Tried to dereference a file that has no references", url); + } + + if (references > 0) { + return this.references.set(url, references); + } + + this.references.delete(url); + + // Drop "storage/" from url and join it with full storage path + const filePath = path.join(helper.getStoragePath(), url.substring(8)); + + fs.unlink(filePath, (err) => { + if (err) { + log.error("Failed to delete stored file", err); + } + }); + } + + store(data, extension, callback) { + const hash = crypto.createHash("sha256").update(data).digest("hex"); + const a = hash.substring(0, 2); + const b = hash.substring(2, 4); + const folder = path.join(helper.getStoragePath(), a, b); + const filePath = path.join(folder, `${hash.substring(4)}.${extension}`); + const url = `storage/${a}/${b}/${hash.substring(4)}.${extension}`; + + this.references.set(url, 1 + (this.references.get(url) || 0)); + + // If file with this name already exists, we don't need to write it again + if (fs.existsSync(filePath)) { + return callback(url); + } + + fsextra.ensureDir(folder).then(() => { + fs.writeFile(filePath, data, (err) => { + if (err) { + log.error("Failed to store a file", err); + + return callback(""); + } + + callback(url); + }); + }).catch((err) => { + log.error("Failed to create storage folder", err); + + return callback(""); + }); + } +} + +module.exports = new Storage(); diff --git a/src/server.js b/src/server.js index 6465af3c..3bde88ee 100644 --- a/src/server.js +++ b/src/server.js @@ -33,6 +33,10 @@ module.exports = function() { .use(allRequests) .use(index) .use(express.static("client")) + .use("/storage/", express.static(Helper.getStoragePath(), { + redirect: false, + maxAge: 86400 * 1000, + })) .engine("html", expressHandlebars({ extname: ".html", helpers: { @@ -152,7 +156,24 @@ function index(req, res, next) { filename: filename }; }); - res.setHeader("Content-Security-Policy", "default-src *; connect-src 'self' ws: wss:; style-src * 'unsafe-inline'; script-src 'self'; child-src 'self'; object-src 'none'; form-action 'none';"); + + const policies = [ + "default-src *", + "connect-src 'self' ws: wss:", + "style-src * 'unsafe-inline'", + "script-src 'self'", + "child-src 'self'", + "object-src 'none'", + "form-action 'none'", + ]; + + // If prefetch is enabled, but storage is not, we have to allow mixed content + if (Helper.config.prefetchStorage || !Helper.config.prefetch) { + policies.push("img-src 'self'"); + policies.unshift("block-all-mixed-content"); + } + + res.setHeader("Content-Security-Policy", policies.join("; ")); res.setHeader("Referrer-Policy", "no-referrer"); res.render("index", data); } diff --git a/test/plugins/link.js b/test/plugins/link.js index e0a0b0ea..3f2aa9bc 100644 --- a/test/plugins/link.js +++ b/test/plugins/link.js @@ -1,10 +1,10 @@ "use strict"; -const expect = require("chai").expect; - -var util = require("../util"); -var link = require("../../src/plugins/irc-events/link.js"); const path = require("path"); +const expect = require("chai").expect; +const util = require("../util"); +const Helper = require("../../src/helper"); +const link = require("../../src/plugins/irc-events/link.js"); describe("Link plugin", function() { before(function(done) { @@ -22,6 +22,8 @@ describe("Link plugin", function() { beforeEach(function() { this.irc = util.createClient(); this.network = util.createNetwork(); + + Helper.config.prefetchStorage = false; }); it("should be able to fetch basic information about URLs", function(done) { @@ -39,6 +41,7 @@ describe("Link plugin", function() { expect(data.preview.type).to.equal("link"); expect(data.preview.head).to.equal("test title"); expect(data.preview.body).to.equal("simple description"); + expect(data.preview.link).to.equal("http://localhost:9002/basic"); expect(message.previews.length).to.equal(1); done(); }); @@ -104,11 +107,13 @@ describe("Link plugin", function() { link(this.irc, this.network.channels[0], message); this.app.get("/invalid-thumb", function(req, res) { - res.send("test"); + res.send("test invalid image"); }); this.irc.once("msg:preview", function(data) { expect(data.preview.thumb).to.be.empty; + expect(data.preview.head).to.equal("test invalid image"); + expect(data.preview.link).to.equal("http://localhost:9002/invalid-thumb"); done(); }); }); @@ -127,6 +132,7 @@ describe("Link plugin", function() { this.irc.once("msg:preview", function(data) { expect(data.preview.head).to.equal("Untitled page"); expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png"); + expect(data.preview.link).to.equal("http://localhost:9002/thumb-no-title"); done(); }); }); @@ -144,6 +150,7 @@ describe("Link plugin", function() { this.irc.once("msg:preview", function(data) { expect(data.preview.head).to.equal("404 image"); + expect(data.preview.link).to.equal("http://localhost:9002/thumb-404"); expect(data.preview.thumb).to.be.empty; done(); }); @@ -159,6 +166,7 @@ describe("Link plugin", function() { this.irc.once("msg:preview", function(data) { expect(data.preview.type).to.equal("image"); expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png"); + expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png"); done(); }); }); diff --git a/test/plugins/storage.js b/test/plugins/storage.js new file mode 100644 index 00000000..e372fe01 --- /dev/null +++ b/test/plugins/storage.js @@ -0,0 +1,68 @@ +"use strict"; + +const fs = require("fs"); +const path = require("path"); +const crypto = require("crypto"); +const expect = require("chai").expect; +const util = require("../util"); +const Helper = require("../../src/helper"); +const link = require("../../src/plugins/irc-events/link.js"); + +describe("Image storage", function() { + const testImagePath = path.resolve(__dirname, "../../client/img/apple-touch-icon-120x120.png"); + const correctImageHash = crypto.createHash("sha256").update(fs.readFileSync(testImagePath)).digest("hex"); + const correctImageURL = `storage/${correctImageHash.substring(0, 2)}/${correctImageHash.substring(2, 4)}/${correctImageHash.substring(4)}.png`; + + before(function(done) { + this.app = util.createWebserver(); + this.app.get("/real-test-image.png", function(req, res) { + res.sendFile(testImagePath); + }); + this.connection = this.app.listen(9003, done); + }); + + after(function(done) { + this.connection.close(done); + }); + + beforeEach(function() { + this.irc = util.createClient(); + this.network = util.createNetwork(); + + Helper.config.prefetchStorage = true; + }); + + it("should store the thumbnail", function(done) { + const message = this.irc.createMessage({ + text: "http://localhost:9003/thumb" + }); + + link(this.irc, this.network.channels[0], message); + + this.app.get("/thumb", function(req, res) { + res.send("Google"); + }); + + this.irc.once("msg:preview", function(data) { + expect(data.preview.head).to.equal("Google"); + expect(data.preview.link).to.equal("http://localhost:9003/thumb"); + expect(data.preview.thumb).to.equal(correctImageURL); + done(); + }); + }); + + it("should store the image", function(done) { + const message = this.irc.createMessage({ + text: "http://localhost:9003/real-test-image.png" + }); + + link(this.irc, this.network.channels[0], message); + + this.irc.once("msg:preview", function(data) { + expect(data.preview.type).to.equal("image"); + expect(data.preview.link).to.equal("http://localhost:9003/real-test-image.png"); + expect(data.preview.thumb).to.equal(correctImageURL); + done(); + }); + }); +});