Merge pull request #315 from vohof/feat/conserve
impose size restrictions when fetching link
This commit is contained in:
commit
ae1a247aeb
@ -28,14 +28,15 @@
|
|||||||
"bcrypt-nodejs": "0.0.3",
|
"bcrypt-nodejs": "0.0.3",
|
||||||
"cheerio": "^0.17.0",
|
"cheerio": "^0.17.0",
|
||||||
"commander": "^2.3.0",
|
"commander": "^2.3.0",
|
||||||
|
"event-stream": "^3.1.7",
|
||||||
"express": "^4.9.5",
|
"express": "^4.9.5",
|
||||||
"lodash": "~2.4.1",
|
"lodash": "~2.4.1",
|
||||||
"mkdirp": "^0.5.0",
|
"mkdirp": "^0.5.0",
|
||||||
"moment": "~2.7.0",
|
"moment": "~2.7.0",
|
||||||
"read": "^1.0.5",
|
"read": "^1.0.5",
|
||||||
|
"request": "^2.51.0",
|
||||||
"slate-irc": "~0.7.3",
|
"slate-irc": "~0.7.3",
|
||||||
"socket.io": "~1.0.6",
|
"socket.io": "~1.0.6"
|
||||||
"superagent": "^0.18.2"
|
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"grunt": "~0.4.5",
|
"grunt": "~0.4.5",
|
||||||
|
@ -1,8 +1,9 @@
|
|||||||
var _ = require("lodash");
|
var _ = require("lodash");
|
||||||
var cheerio = require("cheerio");
|
var cheerio = require("cheerio");
|
||||||
var Msg = require("../../models/msg");
|
var Msg = require("../../models/msg");
|
||||||
var request = require("superagent");
|
var request = require("request");
|
||||||
var Helper = require("../../helper");
|
var Helper = require("../../helper");
|
||||||
|
var es = require('event-stream');
|
||||||
|
|
||||||
module.exports = function(irc, network) {
|
module.exports = function(irc, network) {
|
||||||
var client = this;
|
var client = this;
|
||||||
@ -60,7 +61,7 @@ function parse(msg, url, res, client) {
|
|||||||
|
|
||||||
switch (res.type) {
|
switch (res.type) {
|
||||||
case "text/html":
|
case "text/html":
|
||||||
var $ = cheerio.load(res.res.text);
|
var $ = cheerio.load(res.text);
|
||||||
toggle.type = "link";
|
toggle.type = "link";
|
||||||
toggle.head = $("title").text();
|
toggle.head = $("title").text();
|
||||||
toggle.body =
|
toggle.body =
|
||||||
@ -89,9 +90,35 @@ function parse(msg, url, res, client) {
|
|||||||
|
|
||||||
function fetch(url, cb) {
|
function fetch(url, cb) {
|
||||||
var req = request.get(url);
|
var req = request.get(url);
|
||||||
req.end(function(e, res) {
|
var length = 0;
|
||||||
if (res) {
|
var limit = 1024 * 10;
|
||||||
cb(res);
|
req
|
||||||
}
|
.on('response', function(res) {
|
||||||
});
|
if (!(/(text\/html|application\/json)/.test(res.headers['content-type']))) {
|
||||||
|
res.req.abort();
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.on('error', function() {})
|
||||||
|
.pipe(es.map(function(data, next) {
|
||||||
|
length += data.length;
|
||||||
|
if (length > limit) {
|
||||||
|
req.response.req.abort();
|
||||||
|
}
|
||||||
|
next(null, data);
|
||||||
|
}))
|
||||||
|
.pipe(es.wait(function(err, data) {
|
||||||
|
if (err) return;
|
||||||
|
var body;
|
||||||
|
try {
|
||||||
|
body = JSON.parse(data);
|
||||||
|
} catch(e) {
|
||||||
|
body = {};
|
||||||
|
}
|
||||||
|
data = {
|
||||||
|
text: data,
|
||||||
|
body: body,
|
||||||
|
type: req.response.headers['content-type'].split(/ *; */).shift()
|
||||||
|
};
|
||||||
|
cb(data);
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user