From bd191c09e5ff0a92fb8901f446b37389f4620178 Mon Sep 17 00:00:00 2001 From: Cyrus Date: Mon, 22 Dec 2014 07:26:55 +0800 Subject: [PATCH 1/4] conserve bandwidth include application/json should be using hyperquest appplication typo adapt res res.text remove superagent dependency --- package.json | 5 +++-- src/plugins/irc-events/link.js | 35 +++++++++++++++++++++++++++++----- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index e9275208..db2ec2f3 100644 --- a/package.json +++ b/package.json @@ -28,14 +28,15 @@ "bcrypt-nodejs": "0.0.3", "cheerio": "^0.17.0", "commander": "^2.3.0", + "event-stream": "^3.1.7", "express": "^4.9.5", + "hyperquest": "^1.0.1", "lodash": "~2.4.1", "mkdirp": "^0.5.0", "moment": "~2.7.0", "read": "^1.0.5", "slate-irc": "~0.7.3", - "socket.io": "~1.0.6", - "superagent": "^0.18.2" + "socket.io": "~1.0.6" }, "devDependencies": { "grunt": "~0.4.5", diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js index 7e3e2fe7..68fcdb7a 100644 --- a/src/plugins/irc-events/link.js +++ b/src/plugins/irc-events/link.js @@ -1,8 +1,9 @@ var _ = require("lodash"); var cheerio = require("cheerio"); var Msg = require("../../models/msg"); -var request = require("superagent"); +var request = require("hyperquest"); var Helper = require("../../helper"); +var es = require('event-stream'); module.exports = function(irc, network) { var client = this; @@ -60,7 +61,7 @@ function parse(msg, url, res, client) { switch (res.type) { case "text/html": - var $ = cheerio.load(res.res.text); + var $ = cheerio.load(res.text); toggle.type = "link"; toggle.head = $("title").text(); toggle.body = @@ -89,9 +90,33 @@ function parse(msg, url, res, client) { function fetch(url, cb) { var req = request.get(url); - req.end(function(e, res) { - if (res) { - cb(res); + var length = 0; + var limit = 1024; + req.on('response', function(res) { + if (!(/(text\/html|application\/json)/.test(res.headers['content-type']))) { + // stop wasting precious bandwidth <3 + res.req.abort(); } }); + req.pipe(es.map(function(data, next) { + length += data.length; + if (length > limit) { + req.response.req.abort(); + } + next(null, data); + })).pipe(es.wait(function(err, data) { + var body; + try { + body = JSON.parse(data); + } catch(e) { + console.log(data, e); + body = {}; + } + data = { + text: data, + body: body, + type: req.response.headers['content-type'].split(';').shift() + }; + if (!err) cb(data); + })); } From db6c6aaffcdaf4e693d913f2094f906e7ec47bac Mon Sep 17 00:00:00 2001 From: Cyrus Date: Mon, 22 Dec 2014 08:57:29 +0800 Subject: [PATCH 2/4] unconsole.log --- src/plugins/irc-events/link.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js index 68fcdb7a..f334047a 100644 --- a/src/plugins/irc-events/link.js +++ b/src/plugins/irc-events/link.js @@ -109,7 +109,6 @@ function fetch(url, cb) { try { body = JSON.parse(data); } catch(e) { - console.log(data, e); body = {}; } data = { From 3927ddc0c0355939fe15c528bf249dbeede86404 Mon Sep 17 00:00:00 2001 From: Cyrus Date: Tue, 23 Dec 2014 08:53:57 +0800 Subject: [PATCH 3/4] use request which handles redirects --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index db2ec2f3..aba31fd5 100644 --- a/package.json +++ b/package.json @@ -30,11 +30,11 @@ "commander": "^2.3.0", "event-stream": "^3.1.7", "express": "^4.9.5", - "hyperquest": "^1.0.1", "lodash": "~2.4.1", "mkdirp": "^0.5.0", "moment": "~2.7.0", "read": "^1.0.5", + "request": "^2.51.0", "slate-irc": "~0.7.3", "socket.io": "~1.0.6" }, From 36d861fd79745d612f460070a5804bafec97ff6c Mon Sep 17 00:00:00 2001 From: Cyrus Date: Tue, 23 Dec 2014 09:06:11 +0800 Subject: [PATCH 4/4] handles errors, increase limit --- src/plugins/irc-events/link.js | 59 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js index f334047a..c3b22447 100644 --- a/src/plugins/irc-events/link.js +++ b/src/plugins/irc-events/link.js @@ -1,7 +1,7 @@ var _ = require("lodash"); var cheerio = require("cheerio"); var Msg = require("../../models/msg"); -var request = require("hyperquest"); +var request = require("request"); var Helper = require("../../helper"); var es = require('event-stream'); @@ -91,31 +91,34 @@ function parse(msg, url, res, client) { function fetch(url, cb) { var req = request.get(url); var length = 0; - var limit = 1024; - req.on('response', function(res) { - if (!(/(text\/html|application\/json)/.test(res.headers['content-type']))) { - // stop wasting precious bandwidth <3 - res.req.abort(); - } - }); - req.pipe(es.map(function(data, next) { - length += data.length; - if (length > limit) { - req.response.req.abort(); - } - next(null, data); - })).pipe(es.wait(function(err, data) { - var body; - try { - body = JSON.parse(data); - } catch(e) { - body = {}; - } - data = { - text: data, - body: body, - type: req.response.headers['content-type'].split(';').shift() - }; - if (!err) cb(data); - })); + var limit = 1024 * 10; + req + .on('response', function(res) { + if (!(/(text\/html|application\/json)/.test(res.headers['content-type']))) { + res.req.abort(); + } + }) + .on('error', function() {}) + .pipe(es.map(function(data, next) { + length += data.length; + if (length > limit) { + req.response.req.abort(); + } + next(null, data); + })) + .pipe(es.wait(function(err, data) { + if (err) return; + var body; + try { + body = JSON.parse(data); + } catch(e) { + body = {}; + } + data = { + text: data, + body: body, + type: req.response.headers['content-type'].split(/ *; */).shift() + }; + cb(data); + })); }