De-duplicate link fetching to one request at once
This commit is contained in:
parent
af11b864ce
commit
4e2523060b
@ -8,7 +8,7 @@ const Helper = require("../../helper");
|
|||||||
const cleanIrcMessage = require("../../../client/js/libs/handlebars/ircmessageparser/cleanIrcMessage");
|
const cleanIrcMessage = require("../../../client/js/libs/handlebars/ircmessageparser/cleanIrcMessage");
|
||||||
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
|
const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks");
|
||||||
const storage = require("../storage");
|
const storage = require("../storage");
|
||||||
|
const currentFetchPromises = new Map();
|
||||||
const mediaTypeRegex = /^(audio|video)\/.+/;
|
const mediaTypeRegex = /^(audio|video)\/.+/;
|
||||||
|
|
||||||
// Fix ECDH curve client compatibility in Node v8/v9
|
// Fix ECDH curve client compatibility in Node v8/v9
|
||||||
@ -63,19 +63,13 @@ module.exports = function(client, chan, msg) {
|
|||||||
fetch(url, {
|
fetch(url, {
|
||||||
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
language: client.language,
|
language: client.language,
|
||||||
}, function(res, err) {
|
}).then((res) => {
|
||||||
if (err) {
|
parse(msg, preview, res, client);
|
||||||
|
}).catch((err) => {
|
||||||
preview.type = "error";
|
preview.type = "error";
|
||||||
preview.error = "message";
|
preview.error = "message";
|
||||||
preview.message = err.message;
|
preview.message = err.message;
|
||||||
handlePreview(client, msg, preview, res);
|
handlePreview(client, msg, preview, null);
|
||||||
}
|
|
||||||
|
|
||||||
if (res === null) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
parse(msg, preview, res, client);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
return cleanLinks;
|
return cleanLinks;
|
||||||
@ -111,7 +105,7 @@ function parseHtml(preview, res, client) {
|
|||||||
|
|
||||||
// Verify that thumbnail pic exists and is under allowed size
|
// Verify that thumbnail pic exists and is under allowed size
|
||||||
if (preview.thumb.length) {
|
if (preview.thumb.length) {
|
||||||
fetch(preview.thumb, {language: client.language}, (resThumb) => {
|
fetch(preview.thumb, {language: client.language}).then((resThumb) => {
|
||||||
if (resThumb === null
|
if (resThumb === null
|
||||||
|| !(/^image\/.+/.test(resThumb.type))
|
|| !(/^image\/.+/.test(resThumb.type))
|
||||||
|| resThumb.size > (Helper.config.prefetchMaxImageSize * 1024)) {
|
|| resThumb.size > (Helper.config.prefetchMaxImageSize * 1024)) {
|
||||||
@ -119,6 +113,9 @@ function parseHtml(preview, res, client) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
resolve(resThumb);
|
resolve(resThumb);
|
||||||
|
}).catch(() => {
|
||||||
|
preview.thumb = "";
|
||||||
|
resolve(null);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
resolve(res);
|
resolve(res);
|
||||||
@ -158,7 +155,7 @@ function parseHtmlMedia($, preview, res, client) {
|
|||||||
"video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5" :
|
"video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5" :
|
||||||
"audio/webm, audio/ogg, audio/wav, audio/*;q=0.9, application/ogg;q=0.7, video/*;q=0.6; */*;q=0.5",
|
"audio/webm, audio/ogg, audio/wav, audio/*;q=0.9, application/ogg;q=0.7, video/*;q=0.6; */*;q=0.5",
|
||||||
language: client.language,
|
language: client.language,
|
||||||
}, (resMedia) => {
|
}).then((resMedia) => {
|
||||||
if (resMedia === null || !mediaTypeRegex.test(resMedia.type)) {
|
if (resMedia === null || !mediaTypeRegex.test(resMedia.type)) {
|
||||||
return reject();
|
return reject();
|
||||||
}
|
}
|
||||||
@ -168,7 +165,7 @@ function parseHtmlMedia($, preview, res, client) {
|
|||||||
preview.mediaType = resMedia.type;
|
preview.mediaType = resMedia.type;
|
||||||
|
|
||||||
resolve(resMedia);
|
resolve(resMedia);
|
||||||
});
|
}).catch(reject);
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -313,7 +310,16 @@ function getRequestHeaders(headers) {
|
|||||||
return formattedHeaders;
|
return formattedHeaders;
|
||||||
}
|
}
|
||||||
|
|
||||||
function fetch(uri, headers, cb) {
|
function fetch(uri, headers) {
|
||||||
|
// Stringify the object otherwise the objects won't compute to the same value
|
||||||
|
const cacheKey = JSON.stringify([uri, headers]);
|
||||||
|
let promise = currentFetchPromises.get(cacheKey);
|
||||||
|
|
||||||
|
if (promise) {
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
promise = new Promise((resolve, reject) => {
|
||||||
let req;
|
let req;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@ -324,7 +330,7 @@ function fetch(uri, headers, cb) {
|
|||||||
headers: getRequestHeaders(headers),
|
headers: getRequestHeaders(headers),
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
return cb(null, e);
|
return reject(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
const buffers = [];
|
const buffers = [];
|
||||||
@ -350,7 +356,7 @@ function fetch(uri, headers, cb) {
|
|||||||
limit = 1024 * 50;
|
limit = 1024 * 50;
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.on("error", (e) => cb(null, e))
|
.on("error", (e) => reject(e))
|
||||||
.on("data", (data) => {
|
.on("data", (data) => {
|
||||||
length += data.length;
|
length += data.length;
|
||||||
buffers.push(data);
|
buffers.push(data);
|
||||||
@ -361,7 +367,7 @@ function fetch(uri, headers, cb) {
|
|||||||
})
|
})
|
||||||
.on("end", () => {
|
.on("end", () => {
|
||||||
if (req.response.statusCode < 200 || req.response.statusCode > 299) {
|
if (req.response.statusCode < 200 || req.response.statusCode > 299) {
|
||||||
return cb(null, new Error(`HTTP ${req.response.statusCode}`));
|
return reject(new Error(`HTTP ${req.response.statusCode}`));
|
||||||
}
|
}
|
||||||
|
|
||||||
let type = "";
|
let type = "";
|
||||||
@ -376,8 +382,17 @@ function fetch(uri, headers, cb) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const data = Buffer.concat(buffers, length);
|
const data = Buffer.concat(buffers, length);
|
||||||
cb({data, type, size});
|
resolve({data, type, size});
|
||||||
});
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const removeCache = () => currentFetchPromises.delete(cacheKey);
|
||||||
|
|
||||||
|
promise.then(removeCache).catch(removeCache);
|
||||||
|
|
||||||
|
currentFetchPromises.set(cacheKey, promise);
|
||||||
|
|
||||||
|
return promise;
|
||||||
}
|
}
|
||||||
|
|
||||||
function normalizeURL(link, baseLink, disallowHttp = false) {
|
function normalizeURL(link, baseLink, disallowHttp = false) {
|
||||||
|
@ -422,4 +422,85 @@ describe("Link plugin", function() {
|
|||||||
|
|
||||||
expect(message.previews).to.be.empty;
|
expect(message.previews).to.be.empty;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("should fetch same link only once at the same time", function(done) {
|
||||||
|
const message = this.irc.createMessage({
|
||||||
|
text: "http://localhost:9002/basic-og-once",
|
||||||
|
});
|
||||||
|
|
||||||
|
let requests = 0;
|
||||||
|
let responses = 0;
|
||||||
|
|
||||||
|
this.irc.language = "very nice language";
|
||||||
|
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
process.nextTick(() => link(this.irc, this.network.channels[0], message));
|
||||||
|
|
||||||
|
app.get("/basic-og-once", function(req, res) {
|
||||||
|
requests++;
|
||||||
|
|
||||||
|
expect(req.header("accept-language")).to.equal("very nice language");
|
||||||
|
|
||||||
|
// delay the request so it doesn't resolve immediately
|
||||||
|
setTimeout(() => {
|
||||||
|
res.send("<title>test prefetch</title>");
|
||||||
|
}, 100);
|
||||||
|
});
|
||||||
|
|
||||||
|
const cb = (data) => {
|
||||||
|
responses++;
|
||||||
|
|
||||||
|
expect(data.preview.head, "test prefetch");
|
||||||
|
|
||||||
|
if (responses === 3) {
|
||||||
|
this.irc.removeListener("msg:preview", cb);
|
||||||
|
expect(requests).to.equal(1);
|
||||||
|
done();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.irc.on("msg:preview", cb);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("should fetch same link with different languages multiple times", function(done) {
|
||||||
|
const message = this.irc.createMessage({
|
||||||
|
text: "http://localhost:9002/basic-og-once-lang",
|
||||||
|
});
|
||||||
|
|
||||||
|
const requests = [];
|
||||||
|
let responses = 0;
|
||||||
|
|
||||||
|
this.irc.language = "first language";
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
|
||||||
|
this.irc.language = "second language";
|
||||||
|
link(this.irc, this.network.channels[0], message);
|
||||||
|
|
||||||
|
app.get("/basic-og-once-lang", function(req, res) {
|
||||||
|
requests.push(req.header("accept-language"));
|
||||||
|
|
||||||
|
// delay the request so it doesn't resolve immediately
|
||||||
|
setTimeout(() => {
|
||||||
|
res.send("<title>test prefetch</title>");
|
||||||
|
}, 100);
|
||||||
|
});
|
||||||
|
|
||||||
|
const cb = (data) => {
|
||||||
|
responses++;
|
||||||
|
|
||||||
|
expect(data.preview.head, "test prefetch");
|
||||||
|
|
||||||
|
if (responses === 2) {
|
||||||
|
this.irc.removeListener("msg:preview", cb);
|
||||||
|
expect(requests).to.deep.equal([
|
||||||
|
"first language",
|
||||||
|
"second language",
|
||||||
|
]);
|
||||||
|
done();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
this.irc.on("msg:preview", cb);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
Loading…
Reference in New Issue
Block a user