From fe68f2a1eeb46dfce3be3c9921ac766c235645a6 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Mon, 15 Apr 2019 19:19:50 +0300 Subject: [PATCH] Replace `request` with `got` --- package.json | 2 +- scripts/generate-emoji.js | 11 ++- src/plugins/changelog.js | 36 +++++----- src/plugins/irc-events/link.js | 122 ++++++++++++++++----------------- src/server.js | 7 +- test/server.js | 30 ++++---- yarn.lock | 4 +- 7 files changed, 103 insertions(+), 109 deletions(-) diff --git a/package.json b/package.json index 3758f02b..86107871 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "file-type": "11.0.0", "filenamify": "4.0.0", "fs-extra": "8.0.0", + "got": "9.6.0", "irc-framework": "4.3.0", "is-utf8": "0.2.1", "linkify-it": "2.1.0", @@ -55,7 +56,6 @@ "package-json": "6.3.0", "read": "1.0.7", "read-chunk": "3.2.0", - "request": "2.88.0", "semver": "6.0.0", "socket.io": "2.2.0", "thelounge-ldapjs-non-maintained-fork": "1.0.2", diff --git a/scripts/generate-emoji.js b/scripts/generate-emoji.js index 2c927f48..98808177 100644 --- a/scripts/generate-emoji.js +++ b/scripts/generate-emoji.js @@ -1,14 +1,13 @@ "use strict"; -const request = require("request"); +const got = require("got"); const path = require("path"); const fs = require("fs"); const fuzzy = require("fuzzy"); -request.get({ - url: "https://raw.githubusercontent.com/emojione/emojione/master/extras/alpha-codes/eac.json", - json: true, -}, (error, response, emojiStrategy) => { +(async () => { + const response = await got("https://raw.githubusercontent.com/emojione/emojione/master/extras/alpha-codes/eac.json"); + const emojiStrategy = JSON.parse(response.body); const emojiMap = {}; const fullNameEmojiMap = {}; @@ -57,7 +56,7 @@ request.get({ "libs", "fullnamemap.json" )), fullNameEmojiMapOutput); -}); +})(); function stringToUnicode(key) { return key diff --git a/src/plugins/changelog.js b/src/plugins/changelog.js index fec9f8fb..8f91cd85 100644 --- a/src/plugins/changelog.js +++ b/src/plugins/changelog.js @@ -1,7 +1,8 @@ "use strict"; +const got = require("got"); +const log = require("../log"); const pkg = require("../../package.json"); -const request = require("request"); const TIME_TO_LIVE = 15 * 60 * 1000; // 15 minutes, in milliseconds @@ -15,30 +16,29 @@ const versions = { }, }; -function fetch(callback) { +async function fetch() { // Serving information from cache if (versions.current.changelog) { - callback(versions); - return; + return versions; } - request.get({ - uri: "https://api.github.com/repos/thelounge/thelounge/releases", - headers: { - "Accept": "application/vnd.github.v3.html", // Request rendered markdown - "User-Agent": pkg.name + "; +" + pkg.repository.git, // Identify the client - }, - }, (error, response, body) => { - if (error || response.statusCode !== 200) { - callback(versions); - return; + try { + const response = await got("https://api.github.com/repos/thelounge/thelounge/releases", { + headers: { + "Accept": "application/vnd.github.v3.html", // Request rendered markdown + "User-Agent": pkg.name + "; +" + pkg.repository.git, // Identify the client + }, + }); + + if (response.statusCode !== 200) { + return versions; } let i; let release; let prerelease = false; - body = JSON.parse(body); + const body = JSON.parse(response.body); // Find the current release among releases on GitHub for (i = 0; i < body.length; i++) { @@ -78,7 +78,9 @@ function fetch(callback) { delete versions.current.changelog; delete versions.latest; }, TIME_TO_LIVE); + } catch (error) { + log.error(`Failed to fetch changelog: ${error}`); + } - callback(versions); - }); + return versions; } diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js index 99ce1d9b..e2b14f54 100644 --- a/src/plugins/irc-events/link.js +++ b/src/plugins/irc-events/link.js @@ -1,7 +1,7 @@ "use strict"; const cheerio = require("cheerio"); -const request = require("request"); +const got = require("got"); const URL = require("url").URL; const mime = require("mime-types"); const Helper = require("../../helper"); @@ -9,6 +9,7 @@ const cleanIrcMessage = require("../../../client/js/libs/handlebars/ircmessagepa const findLinks = require("../../../client/js/libs/handlebars/ircmessageparser/findLinks"); const storage = require("../storage"); const currentFetchPromises = new Map(); +const imageTypeRegex = /^image\/.+/; const mediaTypeRegex = /^(audio|video)\/.+/; // Fix ECDH curve client compatibility in Node v8/v9 @@ -107,7 +108,7 @@ function parseHtml(preview, res, client) { if (preview.thumb.length) { fetch(preview.thumb, {language: client.language}).then((resThumb) => { if (resThumb === null - || !(/^image\/.+/.test(resThumb.type)) + || !(imageTypeRegex.test(resThumb.type)) || resThumb.size > (Helper.config.prefetchMaxImageSize * 1024)) { preview.thumb = ""; } @@ -324,70 +325,69 @@ function fetch(uri, headers) { } promise = new Promise((resolve, reject) => { - let req; + let buffer = Buffer.from(""); + let request; + let response; + let limit = Helper.config.prefetchMaxImageSize * 1024; + + limit = 10240; try { - req = request.get({ - url: uri, - maxRedirects: 5, - timeout: 5000, - headers: getRequestHeaders(headers), - }); + got + .stream(uri, { + timeout: 5000, + headers: getRequestHeaders(headers), + rejectUnauthorized: false, + }) + .on("request", (req) => request = req) + .on("response", function(res) { + response = res; + + if (imageTypeRegex.test(res.headers["content-type"])) { + // response is an image + // if Content-Length header reports a size exceeding the prefetch limit, abort fetch + const contentLength = parseInt(res.headers["content-length"], 10) || 0; + + if (contentLength > limit) { + request.abort(); + } + } else if (mediaTypeRegex.test(res.headers["content-type"])) { + // We don't need to download the file any further after we received content-type header + request.abort(); + } else { + // if not image, limit download to 50kb, since we need only meta tags + // twitter.com sends opengraph meta tags within ~20kb of data for individual tweets + limit = 1024 * 50; + } + }) + .on("error", (e) => reject(e)) + .on("data", (data) => { + buffer = Buffer.concat( + [buffer, data], + buffer.length + data.length + ); + + if (buffer.length >= limit) { + request.abort(); + } + }) + .on("end", () => { + let type = ""; + let size = parseInt(response.headers["content-length"], 10) || buffer.length; + + if (size < buffer.length) { + size = buffer.length; + } + + if (response.headers["content-type"]) { + type = response.headers["content-type"].split(/ *; */).shift(); + } + + resolve({data: buffer, type, size}); + }); } catch (e) { return reject(e); } - - const buffers = []; - let length = 0; - let limit = Helper.config.prefetchMaxImageSize * 1024; - - req - .on("response", function(res) { - if (/^image\/.+/.test(res.headers["content-type"])) { - // response is an image - // if Content-Length header reports a size exceeding the prefetch limit, abort fetch - const contentLength = parseInt(res.headers["content-length"], 10) || 0; - - if (contentLength > limit) { - req.abort(); - } - } else if (mediaTypeRegex.test(res.headers["content-type"])) { - // We don't need to download the file any further after we received content-type header - req.abort(); - } else { - // if not image, limit download to 50kb, since we need only meta tags - // twitter.com sends opengraph meta tags within ~20kb of data for individual tweets - limit = 1024 * 50; - } - }) - .on("error", (e) => reject(e)) - .on("data", (data) => { - length += data.length; - buffers.push(data); - - if (length > limit) { - req.abort(); - } - }) - .on("end", () => { - if (req.response.statusCode < 200 || req.response.statusCode > 299) { - return reject(new Error(`HTTP ${req.response.statusCode}`)); - } - - let type = ""; - let size = parseInt(req.response.headers["content-length"], 10) || length; - - if (size < length) { - size = length; - } - - if (req.response.headers["content-type"]) { - type = req.response.headers["content-type"].split(/ *; */).shift(); - } - - const data = Buffer.concat(buffers, length); - resolve({data, type, size}); - }); }); const removeCache = () => currentFetchPromises.delete(cacheKey); diff --git a/src/server.js b/src/server.js index e5c11039..54fa7455 100644 --- a/src/server.js +++ b/src/server.js @@ -431,10 +431,9 @@ function initializeClient(socket, client, token, lastMessage) { } }); - socket.on("changelog", () => { - changelog.fetch((data) => { - socket.emit("changelog", data); - }); + socket.on("changelog", async () => { + const data = await changelog.fetch(); + socket.emit("changelog", data); }); socket.on("msg:preview:toggle", (data) => { diff --git a/test/server.js b/test/server.js index c5cd026f..8f8f18be 100644 --- a/test/server.js +++ b/test/server.js @@ -3,7 +3,7 @@ const log = require("../src/log"); const Helper = require("../src/helper"); const expect = require("chai").expect; -const request = require("request"); +const got = require("got"); const io = require("socket.io-client"); describe("Server", function() { @@ -30,26 +30,20 @@ describe("Server", function() { const webURL = `http://${Helper.config.host}:${Helper.config.port}/`; describe("Express", () => { - it("should run a web server on " + webURL, (done) => { - request(webURL, (error, response, body) => { - expect(error).to.be.null; - expect(body).to.include("The Lounge"); - expect(body).to.include("js/bundle.js"); - - done(); - }); + it("should run a web server on " + webURL, async () => { + const response = await got(webURL); + expect(response.statusCode).to.equal(200); + expect(response.body).to.include("The Lounge"); + expect(response.body).to.include("js/bundle.js"); }); - it("should serve static content correctly", (done) => { - request(webURL + "thelounge.webmanifest", (error, response, body) => { - expect(error).to.be.null; + it("should serve static content correctly", async () => { + const response = await got(webURL + "thelounge.webmanifest"); + const body = JSON.parse(response.body); - body = JSON.parse(body); - expect(body.name).to.equal("The Lounge"); - expect(response.headers["content-type"]).to.equal("application/manifest+json"); - - done(); - }); + expect(response.statusCode).to.equal(200); + expect(body.name).to.equal("The Lounge"); + expect(response.headers["content-type"]).to.equal("application/manifest+json"); }); }); diff --git a/yarn.lock b/yarn.lock index fc27f8d4..6ce5fab2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3886,7 +3886,7 @@ gonzales-pe@^4.2.3: dependencies: minimist "1.1.x" -got@^9.6.0: +got@9.6.0, got@^9.6.0: version "9.6.0" resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" integrity sha512-R7eWptXuGYxwijs0eV+v3o6+XH1IqVK8dJOEecQfTmkncw9AV4dcw/Dhxi8MdlqPthxxpZyizMzyg8RTmEsG+Q== @@ -7261,7 +7261,7 @@ replace-ext@1.0.0: resolved "https://registry.yarnpkg.com/replace-ext/-/replace-ext-1.0.0.tgz#de63128373fcbf7c3ccfa4de5a480c45a67958eb" integrity sha1-3mMSg3P8v3w8z6TeWkgMRaZ5WOs= -request@2.88.0, request@^2.87.0: +request@^2.87.0: version "2.88.0" resolved "https://registry.yarnpkg.com/request/-/request-2.88.0.tgz#9c2fca4f7d35b592efe57c7f0a55e81052124fef" integrity sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==