Store preview images on disk for privacy, security and caching
This commit is contained in:
parent
ce0e460368
commit
f35a2809a7
1
.gitignore
vendored
1
.gitignore
vendored
@ -4,6 +4,7 @@ package-lock.json
|
|||||||
|
|
||||||
.nyc_output/
|
.nyc_output/
|
||||||
coverage/
|
coverage/
|
||||||
|
test/fixtures/.lounge/storage/
|
||||||
|
|
||||||
# Built assets created at npm install/prepublish time
|
# Built assets created at npm install/prepublish time
|
||||||
# See https://docs.npmjs.com/misc/scripts
|
# See https://docs.npmjs.com/misc/scripts
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
<div class="toggle-content toggle-type-{{type}}{{#if shown}} show{{/if}}">
|
<div class="toggle-content toggle-type-{{type}}{{#if shown}} show{{/if}}">
|
||||||
{{#equal type "image"}}
|
{{#equal type "image"}}
|
||||||
<a class="toggle-thumbnail" href="{{link}}" target="_blank" rel="noopener">
|
<a class="toggle-thumbnail" href="{{link}}" target="_blank" rel="noopener">
|
||||||
<img src="{{link}}">
|
<img src="{{thumb}}">
|
||||||
</a>
|
</a>
|
||||||
{{else}}
|
{{else}}
|
||||||
{{#if thumb}}
|
{{#if thumb}}
|
||||||
|
@ -66,6 +66,23 @@ module.exports = {
|
|||||||
//
|
//
|
||||||
prefetch: false,
|
prefetch: false,
|
||||||
|
|
||||||
|
//
|
||||||
|
// Store and proxy prefetched images and thumbnails.
|
||||||
|
// This improves security and privacy by not exposing client IP address,
|
||||||
|
// and always loading images from The Lounge instance and making all assets secure,
|
||||||
|
// which in result fixes mixed content warnings.
|
||||||
|
//
|
||||||
|
// If storage is enabled, The Lounge will fetch and store images and thumbnails
|
||||||
|
// in ~/.lounge/storage folder, or %HOME%/storage if --home is used.
|
||||||
|
//
|
||||||
|
// Images are deleted when they are no longer referenced by any message (controlled by maxHistory),
|
||||||
|
// and the folder is cleaned up on every The Lounge restart.
|
||||||
|
//
|
||||||
|
// @type boolean
|
||||||
|
// @default false
|
||||||
|
//
|
||||||
|
prefetchStorage: false,
|
||||||
|
|
||||||
//
|
//
|
||||||
// Prefetch URLs Image Preview size limit
|
// Prefetch URLs Image Preview size limit
|
||||||
//
|
//
|
||||||
|
@ -12,6 +12,7 @@ const colors = require("colors/safe");
|
|||||||
var Helper = {
|
var Helper = {
|
||||||
config: null,
|
config: null,
|
||||||
expandHome: expandHome,
|
expandHome: expandHome,
|
||||||
|
getStoragePath: getStoragePath,
|
||||||
getUserConfigPath: getUserConfigPath,
|
getUserConfigPath: getUserConfigPath,
|
||||||
getUserLogsPath: getUserLogsPath,
|
getUserLogsPath: getUserLogsPath,
|
||||||
setHome: setHome,
|
setHome: setHome,
|
||||||
@ -90,6 +91,10 @@ function getUserLogsPath(name, network) {
|
|||||||
return path.join(this.HOME, "logs", name, network);
|
return path.join(this.HOME, "logs", name, network);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getStoragePath() {
|
||||||
|
return path.join(this.HOME, "storage");
|
||||||
|
}
|
||||||
|
|
||||||
function ip2hex(address) {
|
function ip2hex(address) {
|
||||||
// no ipv6 support
|
// no ipv6 support
|
||||||
if (!net.isIPv4(address)) {
|
if (!net.isIPv4(address)) {
|
||||||
|
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
var _ = require("lodash");
|
var _ = require("lodash");
|
||||||
var Helper = require("../helper");
|
var Helper = require("../helper");
|
||||||
|
const storage = require("../plugins/storage");
|
||||||
|
|
||||||
module.exports = Chan;
|
module.exports = Chan;
|
||||||
|
|
||||||
@ -53,7 +54,15 @@ Chan.prototype.pushMessage = function(client, msg, increasesUnread) {
|
|||||||
this.messages.push(msg);
|
this.messages.push(msg);
|
||||||
|
|
||||||
if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
|
if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
|
||||||
this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
|
const deleted = this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
|
||||||
|
|
||||||
|
if (Helper.config.prefetch && Helper.config.prefetchStorage) {
|
||||||
|
deleted.forEach((deletedMessage) => {
|
||||||
|
if (deletedMessage.preview && deletedMessage.preview.thumb) {
|
||||||
|
storage.dereference(deletedMessage.preview.thumb);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!msg.self && !isOpen) {
|
if (!msg.self && !isOpen) {
|
||||||
|
@ -5,6 +5,7 @@ const request = require("request");
|
|||||||
const Helper = require("../../helper");
|
const Helper = require("../../helper");
|
||||||
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
|
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
|
||||||
const es = require("event-stream");
|
const es = require("event-stream");
|
||||||
|
const storage = require("../storage");
|
||||||
|
|
||||||
process.setMaxListeners(0);
|
process.setMaxListeners(0);
|
||||||
|
|
||||||
@ -49,7 +50,7 @@ function parse(msg, url, res, client) {
|
|||||||
|
|
||||||
switch (res.type) {
|
switch (res.type) {
|
||||||
case "text/html":
|
case "text/html":
|
||||||
var $ = cheerio.load(res.text);
|
var $ = cheerio.load(res.data);
|
||||||
preview.type = "link";
|
preview.type = "link";
|
||||||
preview.head =
|
preview.head =
|
||||||
$("meta[property=\"og:title\"]").attr("content")
|
$("meta[property=\"og:title\"]").attr("content")
|
||||||
@ -78,7 +79,7 @@ function parse(msg, url, res, client) {
|
|||||||
preview.thumb = "";
|
preview.thumb = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
emitPreview(client, msg, preview);
|
handlePreview(client, msg, preview, resThumb);
|
||||||
});
|
});
|
||||||
|
|
||||||
return;
|
return;
|
||||||
@ -90,18 +91,32 @@ function parse(msg, url, res, client) {
|
|||||||
case "image/gif":
|
case "image/gif":
|
||||||
case "image/jpg":
|
case "image/jpg":
|
||||||
case "image/jpeg":
|
case "image/jpeg":
|
||||||
if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) {
|
if (res.size > (Helper.config.prefetchMaxImageSize * 1024)) {
|
||||||
preview.type = "image";
|
|
||||||
} else {
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
preview.type = "image";
|
||||||
|
preview.thumb = preview.link;
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
handlePreview(client, msg, preview, res);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handlePreview(client, msg, preview, res) {
|
||||||
|
if (!preview.thumb.length || !Helper.config.prefetchStorage) {
|
||||||
|
return emitPreview(client, msg, preview);
|
||||||
|
}
|
||||||
|
|
||||||
|
storage.store(res.data, res.type.replace("image/", ""), (url) => {
|
||||||
|
preview.thumb = url;
|
||||||
|
|
||||||
emitPreview(client, msg, preview);
|
emitPreview(client, msg, preview);
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
function emitPreview(client, msg, preview) {
|
function emitPreview(client, msg, preview) {
|
||||||
@ -164,23 +179,23 @@ function fetch(url, cb) {
|
|||||||
return cb(null);
|
return cb(null);
|
||||||
}
|
}
|
||||||
|
|
||||||
let type;
|
let type = "";
|
||||||
let size = parseInt(req.response.headers["content-length"], 10) || length;
|
let size = parseInt(req.response.headers["content-length"], 10) || length;
|
||||||
|
|
||||||
if (size < length) {
|
if (size < length) {
|
||||||
size = length;
|
size = length;
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
if (req.response.headers["content-type"]) {
|
||||||
type = req.response.headers["content-type"].split(/ *; */).shift();
|
type = req.response.headers["content-type"].split(/ *; */).shift();
|
||||||
} catch (e) {
|
|
||||||
type = {};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
data = {
|
data = {
|
||||||
text: data,
|
data: data,
|
||||||
type: type,
|
type: type,
|
||||||
size: size
|
size: size
|
||||||
};
|
};
|
||||||
|
|
||||||
cb(data);
|
cb(data);
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
81
src/plugins/storage.js
Normal file
81
src/plugins/storage.js
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
"use strict";
|
||||||
|
|
||||||
|
const fs = require("fs");
|
||||||
|
const fsextra = require("fs-extra");
|
||||||
|
const path = require("path");
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const helper = require("../helper");
|
||||||
|
|
||||||
|
class Storage {
|
||||||
|
constructor() {
|
||||||
|
this.references = new Map();
|
||||||
|
|
||||||
|
// Ensures that a directory is empty.
|
||||||
|
// Deletes directory contents if the directory is not empty.
|
||||||
|
// If the directory does not exist, it is created.
|
||||||
|
fsextra.emptyDirSync(helper.getStoragePath());
|
||||||
|
}
|
||||||
|
|
||||||
|
dereference(url) {
|
||||||
|
// If maxHistory is 0, image would be dereferenced before client had a chance to retrieve it,
|
||||||
|
// so for now, just don't implement dereferencing for this edge case.
|
||||||
|
if (helper.maxHistory === 0) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const references = (this.references.get(url) || 0) - 1;
|
||||||
|
|
||||||
|
if (references < 0) {
|
||||||
|
return log.warn("Tried to dereference a file that has no references", url);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (references > 0) {
|
||||||
|
return this.references.set(url, references);
|
||||||
|
}
|
||||||
|
|
||||||
|
this.references.delete(url);
|
||||||
|
|
||||||
|
// Drop "storage/" from url and join it with full storage path
|
||||||
|
const filePath = path.join(helper.getStoragePath(), url.substring(8));
|
||||||
|
|
||||||
|
fs.unlink(filePath, (err) => {
|
||||||
|
if (err) {
|
||||||
|
log.error("Failed to delete stored file", err);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
store(data, extension, callback) {
|
||||||
|
const hash = crypto.createHash("sha256").update(data).digest("hex");
|
||||||
|
const a = hash.substring(0, 2);
|
||||||
|
const b = hash.substring(2, 4);
|
||||||
|
const folder = path.join(helper.getStoragePath(), a, b);
|
||||||
|
const filePath = path.join(folder, `${hash.substring(4)}.${extension}`);
|
||||||
|
const url = `storage/${a}/${b}/${hash.substring(4)}.${extension}`;
|
||||||
|
|
||||||
|
this.references.set(url, 1 + (this.references.get(url) || 0));
|
||||||
|
|
||||||
|
// If file with this name already exists, we don't need to write it again
|
||||||
|
if (fs.existsSync(filePath)) {
|
||||||
|
return callback(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
fsextra.ensureDir(folder).then(() => {
|
||||||
|
fs.writeFile(filePath, data, (err) => {
|
||||||
|
if (err) {
|
||||||
|
log.error("Failed to store a file", err);
|
||||||
|
|
||||||
|
return callback("");
|
||||||
|
}
|
||||||
|
|
||||||
|
callback(url);
|
||||||
|
});
|
||||||
|
}).catch((err) => {
|
||||||
|
log.error("Failed to create storage folder", err);
|
||||||
|
|
||||||
|
return callback("");
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = new Storage();
|
@ -33,6 +33,10 @@ module.exports = function() {
|
|||||||
.use(allRequests)
|
.use(allRequests)
|
||||||
.use(index)
|
.use(index)
|
||||||
.use(express.static("client"))
|
.use(express.static("client"))
|
||||||
|
.use("/storage/", express.static(Helper.getStoragePath(), {
|
||||||
|
redirect: false,
|
||||||
|
maxAge: 86400 * 1000,
|
||||||
|
}))
|
||||||
.engine("html", expressHandlebars({
|
.engine("html", expressHandlebars({
|
||||||
extname: ".html",
|
extname: ".html",
|
||||||
helpers: {
|
helpers: {
|
||||||
@ -152,7 +156,24 @@ function index(req, res, next) {
|
|||||||
filename: filename
|
filename: filename
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
res.setHeader("Content-Security-Policy", "default-src *; connect-src 'self' ws: wss:; style-src * 'unsafe-inline'; script-src 'self'; child-src 'self'; object-src 'none'; form-action 'none';");
|
|
||||||
|
const policies = [
|
||||||
|
"default-src *",
|
||||||
|
"connect-src 'self' ws: wss:",
|
||||||
|
"style-src * 'unsafe-inline'",
|
||||||
|
"script-src 'self'",
|
||||||
|
"child-src 'self'",
|
||||||
|
"object-src 'none'",
|
||||||
|
"form-action 'none'",
|
||||||
|
];
|
||||||
|
|
||||||
|
// If prefetch is enabled, but storage is not, we have to allow mixed content
|
||||||
|
if (Helper.config.prefetchStorage || !Helper.config.prefetch) {
|
||||||
|
policies.push("img-src 'self'");
|
||||||
|
policies.unshift("block-all-mixed-content");
|
||||||
|
}
|
||||||
|
|
||||||
|
res.setHeader("Content-Security-Policy", policies.join("; "));
|
||||||
res.setHeader("Referrer-Policy", "no-referrer");
|
res.setHeader("Referrer-Policy", "no-referrer");
|
||||||
res.render("index", data);
|
res.render("index", data);
|
||||||
}
|
}
|
||||||
|
@ -1,10 +1,10 @@
|
|||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
const expect = require("chai").expect;
|
|
||||||
|
|
||||||
var util = require("../util");
|
|
||||||
var link = require("../../src/plugins/irc-events/link.js");
|
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
|
const expect = require("chai").expect;
|
||||||
|
const util = require("../util");
|
||||||
|
const Helper = require("../../src/helper");
|
||||||
|
const link = require("../../src/plugins/irc-events/link.js");
|
||||||
|
|
||||||
describe("Link plugin", function() {
|
describe("Link plugin", function() {
|
||||||
before(function(done) {
|
before(function(done) {
|
||||||
@ -22,6 +22,8 @@ describe("Link plugin", function() {
|
|||||||
beforeEach(function() {
|
beforeEach(function() {
|
||||||
this.irc = util.createClient();
|
this.irc = util.createClient();
|
||||||
this.network = util.createNetwork();
|
this.network = util.createNetwork();
|
||||||
|
|
||||||
|
Helper.config.prefetchStorage = false;
|
||||||
});
|
});
|
||||||
|
|
||||||
it("should be able to fetch basic information about URLs", function(done) {
|
it("should be able to fetch basic information about URLs", function(done) {
|
||||||
@ -39,6 +41,7 @@ describe("Link plugin", function() {
|
|||||||
expect(data.preview.type).to.equal("link");
|
expect(data.preview.type).to.equal("link");
|
||||||
expect(data.preview.head).to.equal("test title");
|
expect(data.preview.head).to.equal("test title");
|
||||||
expect(data.preview.body).to.equal("simple description");
|
expect(data.preview.body).to.equal("simple description");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9002/basic");
|
||||||
expect(message.previews.length).to.equal(1);
|
expect(message.previews.length).to.equal(1);
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
@ -104,11 +107,13 @@ describe("Link plugin", function() {
|
|||||||
link(this.irc, this.network.channels[0], message);
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
|
||||||
this.app.get("/invalid-thumb", function(req, res) {
|
this.app.get("/invalid-thumb", function(req, res) {
|
||||||
res.send("<title>test</title><meta property='og:image' content='/real-test-image.png'>");
|
res.send("<title>test invalid image</title><meta property='og:image' content='/real-test-image.png'>");
|
||||||
});
|
});
|
||||||
|
|
||||||
this.irc.once("msg:preview", function(data) {
|
this.irc.once("msg:preview", function(data) {
|
||||||
expect(data.preview.thumb).to.be.empty;
|
expect(data.preview.thumb).to.be.empty;
|
||||||
|
expect(data.preview.head).to.equal("test invalid image");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9002/invalid-thumb");
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -127,6 +132,7 @@ describe("Link plugin", function() {
|
|||||||
this.irc.once("msg:preview", function(data) {
|
this.irc.once("msg:preview", function(data) {
|
||||||
expect(data.preview.head).to.equal("Untitled page");
|
expect(data.preview.head).to.equal("Untitled page");
|
||||||
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
|
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9002/thumb-no-title");
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@ -144,6 +150,7 @@ describe("Link plugin", function() {
|
|||||||
|
|
||||||
this.irc.once("msg:preview", function(data) {
|
this.irc.once("msg:preview", function(data) {
|
||||||
expect(data.preview.head).to.equal("404 image");
|
expect(data.preview.head).to.equal("404 image");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9002/thumb-404");
|
||||||
expect(data.preview.thumb).to.be.empty;
|
expect(data.preview.thumb).to.be.empty;
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
@ -159,6 +166,7 @@ describe("Link plugin", function() {
|
|||||||
this.irc.once("msg:preview", function(data) {
|
this.irc.once("msg:preview", function(data) {
|
||||||
expect(data.preview.type).to.equal("image");
|
expect(data.preview.type).to.equal("image");
|
||||||
expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png");
|
expect(data.preview.link).to.equal("http://localhost:9002/real-test-image.png");
|
||||||
|
expect(data.preview.thumb).to.equal("http://localhost:9002/real-test-image.png");
|
||||||
done();
|
done();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
68
test/plugins/storage.js
Normal file
68
test/plugins/storage.js
Normal file
@ -0,0 +1,68 @@
|
|||||||
|
"use strict";
|
||||||
|
|
||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
const crypto = require("crypto");
|
||||||
|
const expect = require("chai").expect;
|
||||||
|
const util = require("../util");
|
||||||
|
const Helper = require("../../src/helper");
|
||||||
|
const link = require("../../src/plugins/irc-events/link.js");
|
||||||
|
|
||||||
|
describe("Image storage", function() {
|
||||||
|
const testImagePath = path.resolve(__dirname, "../../client/img/apple-touch-icon-120x120.png");
|
||||||
|
const correctImageHash = crypto.createHash("sha256").update(fs.readFileSync(testImagePath)).digest("hex");
|
||||||
|
const correctImageURL = `storage/${correctImageHash.substring(0, 2)}/${correctImageHash.substring(2, 4)}/${correctImageHash.substring(4)}.png`;
|
||||||
|
|
||||||
|
before(function(done) {
|
||||||
|
this.app = util.createWebserver();
|
||||||
|
this.app.get("/real-test-image.png", function(req, res) {
|
||||||
|
res.sendFile(testImagePath);
|
||||||
|
});
|
||||||
|
this.connection = this.app.listen(9003, done);
|
||||||
|
});
|
||||||
|
|
||||||
|
after(function(done) {
|
||||||
|
this.connection.close(done);
|
||||||
|
});
|
||||||
|
|
||||||
|
beforeEach(function() {
|
||||||
|
this.irc = util.createClient();
|
||||||
|
this.network = util.createNetwork();
|
||||||
|
|
||||||
|
Helper.config.prefetchStorage = true;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should store the thumbnail", function(done) {
|
||||||
|
const message = this.irc.createMessage({
|
||||||
|
text: "http://localhost:9003/thumb"
|
||||||
|
});
|
||||||
|
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
|
||||||
|
this.app.get("/thumb", function(req, res) {
|
||||||
|
res.send("<title>Google</title><meta property='og:image' content='http://localhost:9003/real-test-image.png'>");
|
||||||
|
});
|
||||||
|
|
||||||
|
this.irc.once("msg:preview", function(data) {
|
||||||
|
expect(data.preview.head).to.equal("Google");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9003/thumb");
|
||||||
|
expect(data.preview.thumb).to.equal(correctImageURL);
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should store the image", function(done) {
|
||||||
|
const message = this.irc.createMessage({
|
||||||
|
text: "http://localhost:9003/real-test-image.png"
|
||||||
|
});
|
||||||
|
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
|
||||||
|
this.irc.once("msg:preview", function(data) {
|
||||||
|
expect(data.preview.type).to.equal("image");
|
||||||
|
expect(data.preview.link).to.equal("http://localhost:9003/real-test-image.png");
|
||||||
|
expect(data.preview.thumb).to.equal(correctImageURL);
|
||||||
|
done();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
Loading…
Reference in New Issue
Block a user