diff --git a/.gitignore b/.gitignore
index 3c7c410b..58e6880a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,7 @@ package-lock.json
.nyc_output/
coverage/
+test/fixtures/.lounge/storage/
# Built assets created at npm install/prepublish time
# See https://docs.npmjs.com/misc/scripts
diff --git a/client/views/msg_preview.tpl b/client/views/msg_preview.tpl
index 24f4a599..aa20c363 100644
--- a/client/views/msg_preview.tpl
+++ b/client/views/msg_preview.tpl
@@ -2,7 +2,7 @@
{{#equal type "image"}}
-
+
{{else}}
{{#if thumb}}
diff --git a/defaults/config.js b/defaults/config.js
index 83d4c7e8..9f757e5f 100644
--- a/defaults/config.js
+++ b/defaults/config.js
@@ -66,6 +66,23 @@ module.exports = {
//
prefetch: false,
+ //
+ // Store and proxy prefetched images and thumbnails.
+ // This improves security and privacy by not exposing client IP address,
+ // and always loading images from The Lounge instance and making all assets secure,
+ // which in result fixes mixed content warnings.
+ //
+ // If storage is enabled, The Lounge will fetch and store images and thumbnails
+ // in ~/.lounge/storage folder, or %HOME%/storage if --home is used.
+ //
+ // Images are deleted when they are no longer referenced by any message (controlled by maxHistory),
+ // and the folder is cleaned up on every The Lounge restart.
+ //
+ // @type boolean
+ // @default false
+ //
+ prefetchStorage: false,
+
//
// Prefetch URLs Image Preview size limit
//
diff --git a/src/helper.js b/src/helper.js
index 23c4abfb..2fae848e 100644
--- a/src/helper.js
+++ b/src/helper.js
@@ -12,6 +12,7 @@ const colors = require("colors/safe");
var Helper = {
config: null,
expandHome: expandHome,
+ getStoragePath: getStoragePath,
getUserConfigPath: getUserConfigPath,
getUserLogsPath: getUserLogsPath,
setHome: setHome,
@@ -90,6 +91,10 @@ function getUserLogsPath(name, network) {
return path.join(this.HOME, "logs", name, network);
}
+function getStoragePath() {
+ return path.join(this.HOME, "storage");
+}
+
function ip2hex(address) {
// no ipv6 support
if (!net.isIPv4(address)) {
diff --git a/src/models/chan.js b/src/models/chan.js
index 3ac014ae..66418195 100644
--- a/src/models/chan.js
+++ b/src/models/chan.js
@@ -2,6 +2,7 @@
var _ = require("lodash");
var Helper = require("../helper");
+const storage = require("../plugins/storage");
module.exports = Chan;
@@ -53,7 +54,15 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
this.messages.push(msg);
if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
- this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
+ const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
+
+ if (Helper.config.prefetch && Helper.config.prefetchStorage) {
+ deleted.forEach((deletedMessage) => {
+ if (deletedMessage.preview && deletedMessage.preview.thumb) {
+ storage.dereference(deletedMessage.preview.thumb);
+ }
+ });
+ }
}
if (!msg.self && !isOpen) {
diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js
index e0f0120f..fbf00479 100644
--- a/src/plugins/irc-events/link.js
+++ b/src/plugins/irc-events/link.js
@@ -5,6 +5,7 @@ const request = require("request");
const Helper = require("../../helper");
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
const es = require("event-stream");
+const storage = require("../storage");
process.setMaxListeners(0);
@@ -49,7 +50,7 @@ function parse(msg, url, res, client) {
switch (res.type) {
case "text/html":
- var $ = cheerio.load(res.text);
+ var $ = cheerio.load(res.data);
preview.type = "link";
preview.head =
$("meta[property=\"og:title\"]").attr("content")
@@ -78,7 +79,7 @@ function parse(msg, url, res, client) {
preview.thumb = "";
}
- emitPreview(client, msg, preview);
+ handlePreview(client, msg, preview, resThumb);
});
return;
@@ -90,18 +91,32 @@ function parse(msg, url, res, client) {
case "image/gif":
case "image/jpg":
case "image/jpeg":
- if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) {
- preview.type = "image";
- } else {
+ if (res.size > (Helper.config.prefetchMaxImageSize * 1024)) {
return;
}
+
+ preview.type = "image";
+ preview.thumb = preview.link;
+
break;
default:
return;
}
- emitPreview(client, msg, preview);
+ handlePreview(client, msg, preview, res);
+}
+
+function handlePreview(client, msg, preview, res) {
+ if (!preview.thumb.length || !Helper.config.prefetchStorage) {
+ return emitPreview(client, msg, preview);
+ }
+
+ storage.store(res.data, res.type.replace("image/", ""), (url) => {
+ preview.thumb = url;
+
+ emitPreview(client, msg, preview);
+ });
}
function emitPreview(client, msg, preview) {
@@ -164,23 +179,23 @@ function fetch(url, cb) {
return cb(null);
}
- let type;
+ let type = "";
let size = parseInt(req.response.headers["content-length"], 10) || length;
if (size < length) {
size = length;
}
- try {
+ if (req.response.headers["content-type"]) {
type = req.response.headers["content-type"].split(/ *; */).shift();
- } catch (e) {
- type = {};
}
+
data = {
- text: data,
+ data: data,
type: type,
size: size
};
+
cb(data);
}));
}
diff --git a/src/plugins/storage.js b/src/plugins/storage.js
new file mode 100644
index 00000000..2d889f1b
--- /dev/null
+++ b/src/plugins/storage.js
@@ -0,0 +1,81 @@
+"use strict";
+
+const fs = require("fs");
+const fsextra = require("fs-extra");
+const path = require("path");
+const crypto = require("crypto");
+const helper = require("../helper");
+
+class Storage {
+ constructor() {
+ this.references = new Map();
+
+ // Ensures that a directory is empty.
+ // Deletes directory contents if the directory is not empty.
+ // If the directory does not exist, it is created.
+ fsextra.emptyDirSync(helper.getStoragePath());
+ }
+
+ dereference(url) {
+ // If maxHistory is 0, image would be dereferenced before client had a chance to retrieve it,
+ // so for now, just don't implement dereferencing for this edge case.
+ if (helper.maxHistory === 0) {
+ return;
+ }
+
+ const references = (this.references.get(url) || 0) - 1;
+
+ if (references < 0) {
+ return log.warn("Tried to dereference a file that has no references", url);
+ }
+
+ if (references > 0) {
+ return this.references.set(url, references);
+ }
+
+ this.references.delete(url);
+
+ // Drop "storage/" from url and join it with full storage path
+ const filePath = path.join(helper.getStoragePath(), url.substring(8));
+
+ fs.unlink(filePath, (err) => {
+ if (err) {
+ log.error("Failed to delete stored file", err);
+ }
+ });
+ }
+
+ store(data, extension, callback) {
+ const hash = crypto.createHash("sha256").update(data).digest("hex");
+ const a = hash.substring(0, 2);
+ const b = hash.substring(2, 4);
+ const folder = path.join(helper.getStoragePath(), a, b);
+ const filePath = path.join(folder, `${hash.substring(4)}.${extension}`);
+ const url = `storage/${a}/${b}/${hash.substring(4)}.${extension}`;
+
+ this.references.set(url, 1 + (this.references.get(url) || 0));
+
+ // If file with this name already exists, we don't need to write it again
+ if (fs.existsSync(filePath)) {
+ return callback(url);
+ }
+
+ fsextra.ensureDir(folder).then(() => {
+ fs.writeFile(filePath, data, (err) => {
+ if (err) {
+ log.error("Failed to store a file", err);
+
+ return callback("");
+ }
+
+ callback(url);
+ });
+ }).catch((err) => {
+ log.error("Failed to create storage folder", err);
+
+ return callback("");
+ });
+ }
+}
+
+module.exports = new Storage();
diff --git a/src/server.js b/src/server.js
index 6465af3c..3bde88ee 100644
--- a/src/server.js
+++ b/src/server.js
@@ -33,6 +33,10 @@ module.exports = function() {
.use(allRequests)
.use(index)
.use(express.static("client"))
+ .use("/storage/", express.static(Helper.getStoragePath(), {
+ redirect: false,
+ maxAge: 86400 * 1000,
+ }))
.engine("html", expressHandlebars({
extname: ".html",
helpers: {
@@ -152,7 +156,24 @@ function index(req, res, next) {
filename: filename
};
});
- res.setHeader("Content-Security-Policy", "default-src *; connect-src 'self' ws: wss:; style-src * 'unsafe-inline'; script-src 'self'; child-src 'self'; object-src 'none'; form-action 'none';");
+
+ const policies = [
+ "default-src *",
+ "connect-src 'self' ws: wss:",
+ "style-src * 'unsafe-inline'",
+ "script-src 'self'",
+ "child-src 'self'",
+ "object-src 'none'",
+ "form-action 'none'",
+ ];
+
+ // If prefetch is enabled, but storage is not, we have to allow mixed content
+ if (Helper.config.prefetchStorage || !Helper.config.prefetch) {
+ policies.push("img-src 'self'");
+ policies.unshift("block-all-mixed-content");
+ }
+
+ res.setHeader("Content-Security-Policy", policies.join("; "));
res.setHeader("Referrer-Policy", "no-referrer");
res.render("index", data);
}
diff --git a/test/plugins/link.js b/test/plugins/link.js
index e0a0b0ea..3f2aa9bc 100644
--- a/test/plugins/link.js
+++ b/test/plugins/link.js
@@ -1,10 +1,10 @@
"use strict";
-const expect = require("chai").expect;
-
-var util = require("../util");
-var link = require("../../src/plugins/irc-events/link.js");
const path = require("path");
+const expect = require("chai").expect;
+const util = require("../util");
+const Helper = require("../../src/helper");
+const link = require("../../src/plugins/irc-events/link.js");
describe("Link plugin", function() {
before(function(done) {
@@ -22,6 +22,8 @@ describe("Link plugin", function() {
beforeEach(function() {
this.irc = util.createClient();
this.network = util.createNetwork();
+
+ Helper.config.prefetchStorage = false;
});
it("should be able to fetch basic information about URLs", function(done) {
@@ -39,6 +41,7 @@ describe("Link plugin", function() {
expect(data.preview.type).to.equal("link");
expect(data.preview.head).to.equal("test title");
expect(data.preview.body).to.equal("simple description");
+ expect(data.preview.link).to.equal("http://localhost:9002/basic");
expect(message.previews.length).to.equal(1);
done();
});
@@ -104,11 +107,13 @@ describe("Link plugin", function() {
link(this.irc, this.network.channels[0], message);
this.app.get("/invalid-thumb", function(req, res) {
- res.send("
test");
+ res.send("
test invalid image");
});
this.irc.once("msg:preview", function(data) {
expect(data.preview.thumb).to.be.empty;
+ expect(data.preview.head).to.equal("test invalid image");
+ expect(data.preview.link).to.equal("http://localhost:9002/invalid-thumb");
done();
});
});
@@ -127,6 +132,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.head).to.equal("Untitled page");
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
+ expect(data.preview.link).to.equal("http://localhost:9002/thumb-no-title");
done();
});
});
@@ -144,6 +150,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.head).to.equal("404 image");
+ expect(data.preview.link).to.equal("http://localhost:9002/thumb-404");
expect(data.preview.thumb).to.be.empty;
done();
});
@@ -159,6 +166,7 @@ describe("Link plugin", function() {
this.irc.once("msg:preview", function(data) {
expect(data.preview.type).to.equal("image");
expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png");
+ expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
done();
});
});
diff --git a/test/plugins/storage.js b/test/plugins/storage.js
new file mode 100644
index 00000000..e372fe01
--- /dev/null
+++ b/test/plugins/storage.js
@@ -0,0 +1,68 @@
+"use strict";
+
+const fs = require("fs");
+const path = require("path");
+const crypto = require("crypto");
+const expect = require("chai").expect;
+const util = require("../util");
+const Helper = require("../../src/helper");
+const link = require("../../src/plugins/irc-events/link.js");
+
+describe("Image storage", function() {
+ const testImagePath = path.resolve(__dirname, "../../client/img/apple-touch-icon-120x120.png");
+ const correctImageHash = crypto.createHash("sha256").update(fs.readFileSync(testImagePath)).digest("hex");
+ const correctImageURL = `storage/${correctImageHash.substring(0, 2)}/${correctImageHash.substring(2, 4)}/${correctImageHash.substring(4)}.png`;
+
+ before(function(done) {
+ this.app = util.createWebserver();
+ this.app.get("/real-test-image.png", function(req, res) {
+ res.sendFile(testImagePath);
+ });
+ this.connection = this.app.listen(9003, done);
+ });
+
+ after(function(done) {
+ this.connection.close(done);
+ });
+
+ beforeEach(function() {
+ this.irc = util.createClient();
+ this.network = util.createNetwork();
+
+ Helper.config.prefetchStorage = true;
+ });
+
+ it("should store the thumbnail", function(done) {
+ const message = this.irc.createMessage({
+ text: "http://localhost:9003/thumb"
+ });
+
+ link(this.irc, this.network.channels[0], message);
+
+ this.app.get("/thumb", function(req, res) {
+ res.send("
Google");
+ });
+
+ this.irc.once("msg:preview", function(data) {
+ expect(data.preview.head).to.equal("Google");
+ expect(data.preview.link).to.equal("http://localhost:9003/thumb");
+ expect(data.preview.thumb).to.equal(correctImageURL);
+ done();
+ });
+ });
+
+ it("should store the image", function(done) {
+ const message = this.irc.createMessage({
+ text: "http://localhost:9003/real-test-image.png"
+ });
+
+ link(this.irc, this.network.channels[0], message);
+
+ this.irc.once("msg:preview", function(data) {
+ expect(data.preview.type).to.equal("image");
+ expect(data.preview.link).to.equal("http://localhost:9003/real-test-image.png");
+ expect(data.preview.thumb).to.equal(correctImageURL);
+ done();
+ });
+ });
+});