Merge pull request #2538 from thelounge/xpaw/fetch-links-once

De-duplicate link fetching to one request at once

src/plugins/irc-events/link.js

@@ -8,7 +8,7 @@ const Helper = require("../../helper");
 const cleanIrcMessage = require("../../../client/js/libs/handlebars/ircmessageparser/cleanIrcMessage");
 const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
 const storage = require("../storage");
-
+const currentFetchPromises = new Map();
 const mediaTypeRegex = /^(audio|video)\/.+/;
 
 // Fix ECDH curve client compatibility in Node v8/v9
@@ -63,19 +63,13 @@ module.exports = function(client, chan, msg) {
 		fetch(url, {
 			accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 			language: client.language,
-		}, function(res, err) {
+		}).then((res) => {
-			if (err) {
+			parse(msg, preview, res, client);
+		}).catch((err) => {
 			preview.type = "error";
 			preview.error = "message";
 			preview.message = err.message;
-				handlePreview(client, msg, preview, res);
+			handlePreview(client, msg, preview, null);
-			}
-
-			if (res === null) {
-				return;
-			}
-
-			parse(msg, preview, res, client);
 		});
 
 		return cleanLinks;
@@ -111,7 +105,7 @@ function parseHtml(preview, res, client) {
 
 				// Verify that thumbnail pic exists and is under allowed size
 				if (preview.thumb.length) {
-					fetch(preview.thumb, {language: client.language}, (resThumb) => {
+					fetch(preview.thumb, {language: client.language}).then((resThumb) => {
 						if (resThumb === null
 						|| !(/^image\/.+/.test(resThumb.type))
 						|| resThumb.size > (Helper.config.prefetchMaxImageSize * 1024)) {
@@ -119,6 +113,9 @@ function parseHtml(preview, res, client) {
 						}
 
 						resolve(resThumb);
+					}).catch(() => {
+						preview.thumb = "";
+						resolve(null);
 					});
 				} else {
 					resolve(res);
@@ -158,7 +155,7 @@ function parseHtmlMedia($, preview, res, client) {
 							"video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5" :
 							"audio/webm, audio/ogg, audio/wav, audio/*;q=0.9, application/ogg;q=0.7, video/*;q=0.6; */*;q=0.5",
 						language: client.language,
-					}, (resMedia) => {
+					}).then((resMedia) => {
 						if (resMedia === null || !mediaTypeRegex.test(resMedia.type)) {
 							return reject();
 						}
@@ -168,7 +165,7 @@ function parseHtmlMedia($, preview, res, client) {
 						preview.mediaType = resMedia.type;
 
 						resolve(resMedia);
-					});
+					}).catch(reject);
 
 					return false;
 				}
@@ -313,7 +310,16 @@ function getRequestHeaders(headers) {
 	return formattedHeaders;
 }
 
-function fetch(uri, headers, cb) {
+function fetch(uri, headers) {
+	// Stringify the object otherwise the objects won't compute to the same value
+	const cacheKey = JSON.stringify([uri, headers]);
+	let promise = currentFetchPromises.get(cacheKey);
+
+	if (promise) {
+		return promise;
+	}
+
+	promise = new Promise((resolve, reject) => {
 		let req;
 
 		try {
@@ -324,7 +330,7 @@ function fetch(uri, headers, cb) {
 				headers: getRequestHeaders(headers),
 			});
 		} catch (e) {
-		return cb(null, e);
+			return reject(e);
 		}
 
 		const buffers = [];
@@ -350,7 +356,7 @@ function fetch(uri, headers, cb) {
 					limit = 1024 * 50;
 				}
 			})
-		.on("error", (e) => cb(null, e))
+			.on("error", (e) => reject(e))
 			.on("data", (data) => {
 				length += data.length;
 				buffers.push(data);
@@ -361,7 +367,7 @@ function fetch(uri, headers, cb) {
 			})
 			.on("end", () => {
 				if (req.response.statusCode < 200 || req.response.statusCode > 299) {
-				return cb(null, new Error(`HTTP ${req.response.statusCode}`));
+					return reject(new Error(`HTTP ${req.response.statusCode}`));
 				}
 
 				let type = "";
@@ -376,8 +382,17 @@ function fetch(uri, headers, cb) {
 				}
 
 				const data = Buffer.concat(buffers, length);
-			cb({data, type, size});
+				resolve({data, type, size});
 			});
+	});
+
+	const removeCache = () => currentFetchPromises.delete(cacheKey);
+
+	promise.then(removeCache).catch(removeCache);
+
+	currentFetchPromises.set(cacheKey, promise);
+
+	return promise;
 }
 
 function normalizeURL(link, baseLink, disallowHttp = false) {

test/plugins/link.js

@@ -422,4 +422,85 @@ describe("Link plugin", function() {
 
 		expect(message.previews).to.be.empty;
 	});
+
+	it("should fetch same link only once at the same time", function(done) {
+		const message = this.irc.createMessage({
+			text: "http://localhost:9002/basic-og-once",
+		});
+
+		let requests = 0;
+		let responses = 0;
+
+		this.irc.language = "very nice language";
+
+		link(this.irc, this.network.channels[0], message);
+		link(this.irc, this.network.channels[0], message);
+		process.nextTick(() => link(this.irc, this.network.channels[0], message));
+
+		app.get("/basic-og-once", function(req, res) {
+			requests++;
+
+			expect(req.header("accept-language")).to.equal("very nice language");
+
+			// delay the request so it doesn't resolve immediately
+			setTimeout(() => {
+				res.send("<title>test prefetch</title>");
+			}, 100);
+		});
+
+		const cb = (data) => {
+			responses++;
+
+			expect(data.preview.head, "test prefetch");
+
+			if (responses === 3) {
+				this.irc.removeListener("msg:preview", cb);
+				expect(requests).to.equal(1);
+				done();
+			}
+		};
+
+		this.irc.on("msg:preview", cb);
+	});
+
+	it("should fetch same link with different languages multiple times", function(done) {
+		const message = this.irc.createMessage({
+			text: "http://localhost:9002/basic-og-once-lang",
+		});
+
+		const requests = [];
+		let responses = 0;
+
+		this.irc.language = "first language";
+		link(this.irc, this.network.channels[0], message);
+
+		this.irc.language = "second language";
+		link(this.irc, this.network.channels[0], message);
+
+		app.get("/basic-og-once-lang", function(req, res) {
+			requests.push(req.header("accept-language"));
+
+			// delay the request so it doesn't resolve immediately
+			setTimeout(() => {
+				res.send("<title>test prefetch</title>");
+			}, 100);
+		});
+
+		const cb = (data) => {
+			responses++;
+
+			expect(data.preview.head, "test prefetch");
+
+			if (responses === 2) {
+				this.irc.removeListener("msg:preview", cb);
+				expect(requests).to.deep.equal([
+					"first language",
+					"second language",
+				]);
+				done();
+			}
+		};
+
+		this.irc.on("msg:preview", cb);
+	});
 });