supernets/hardlounge
supernets/hardlounge
4
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3037 from thelounge/xpaw-richrd/file-uploads

Browse Source 
Re-implement file uploading with old school multipart forms

Co-authored-by: Jérémie Astori <jeremie@astori.fr>
This commit is contained in:
Pavel Djundik 2019-03-19 15:09:33 +02:00 committed by GitHub
commit 92098286e7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 397 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
client/components/ChatInput.vue
View File

@ -153,6 +153,7 @@ export default {
},
destroyed() {
require("../js/autocompletion").disable();
upload.abort();
},
methods: {
setPendingMessage(e) {

12
client/css/style.css
View File

@ -30,6 +30,9 @@
/* Background and left-border color of highlight messages */
--highlight-bg-color: #efe8dc;
--highlight-border-color: #b08c4f;
/* Color of the progress bar that appears as a file is being uploaded to the server. Defaults to button color */
--upload-progressbar-color: var(--button-color);
}
/* Samsung Internet <7.0 and Microsoft Edge support (yes, both of them use webkit prefix) */
@ -1956,9 +1959,16 @@ part/quit messages where we don't load previews (adds a blank line otherwise) */
}
#upload-progressbar {
background: blue;
background: var(--upload-progressbar-color);
box-shadow: 0 0 10px var(--upload-progressbar-color);
width: 0%;
height: 2px;
visibility: hidden;
}
#upload-progressbar.upload-progressbar-visible {
visibility: visible;
transition: 0.3s width ease-in-out;
}
#form {

1
client/js/socket-events/index.js
View File

@ -16,7 +16,6 @@ require("./part");
require("./quit");
require("./sync_sort");
require("./topic");
require("./uploads");
require("./users");
require("./sign_out");
require("./sessions_list");

24
client/js/socket-events/uploads.js
View File

@ -1,24 +0,0 @@
"use strict";
const socket = require("../socket");
const updateCursor = require("undate").update;
socket.on("upload:success", (url) => {
const fullURL = (new URL(url, location)).toString();
const textbox = document.getElementById("input");
const initStart = textbox.selectionStart;
// Get the text before the cursor, and add a space if it's not in the beginning
const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : "";
// Get the remaining text after the cursor
const cursorToTail = textbox.value.substr(initStart);
// Construct the value until the point where we want the cursor to be
const textBeforeTail = headToCursor + fullURL + " ";
updateCursor(textbox, textBeforeTail + cursorToTail);
// Set the cursor after the link and a space
textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length;
});

236
client/js/upload.js
View File

@ -1,43 +1,203 @@
"use strict";
const $ = require("jquery");
const socket = require("./socket");
const SocketIOFileUpload = require("socketio-file-upload/client");
const instance = new SocketIOFileUpload(socket);
const {vueApp} = require("./vue");
const updateCursor = require("undate").update;
class Uploader {
init() {
this.xhr = null;
this.fileQueue = [];
this.overlay = document.getElementById("upload-overlay");
this.uploadInput = document.getElementById("upload-input");
this.uploadProgressbar = document.getElementById("upload-progressbar");
this.uploadInput.addEventListener("change", (e) => this.filesChanged(e));
document.addEventListener("dragenter", (e) => this.dragEnter(e));
document.addEventListener("dragover", (e) => this.dragOver(e));
document.addEventListener("dragleave", (e) => this.dragLeave(e));
document.addEventListener("drop", (e) => this.drop(e));
}
dragOver(event) {
// Prevent dragover event completely and do nothing with it
// This stops the browser from trying to guess which cursor to show
event.preventDefault();
}
dragEnter(event) {
event.preventDefault();
// relatedTarget is the target where we entered the drag from
// when dragging from another window, the target is null, otherwise its a DOM element
if (!event.relatedTarget && event.dataTransfer.types.includes("Files")) {
this.overlay.classList.add("is-dragover");
}
}
dragLeave(event) {
event.preventDefault();
// If relatedTarget is null, that means we are no longer dragging over the page
if (!event.relatedTarget) {
this.overlay.classList.remove("is-dragover");
}
}
drop(event) {
event.preventDefault();
this.overlay.classList.remove("is-dragover");
let files;
if (event.dataTransfer.items) {
files = Array.from(event.dataTransfer.items)
.filter((item) => item.kind === "file")
.map((item) => item.getAsFile());
} else {
files = Array.from(event.dataTransfer.files);
}
this.triggerUpload(files);
}
filesChanged() {
const files = Array.from(this.uploadInput.files);
this.triggerUpload(files);
}
triggerUpload(files) {
if (!files.length) {
return;
}
const wasQueueEmpty = this.fileQueue.length === 0;
for (const file of files) {
if (this.maxFileSize > 0 && file.size > this.maxFileSize) {
this.handleResponse({
error: `File ${file.name} is over the maximum allowed size`,
});
continue;
}
this.fileQueue.push(file);
}
// if the queue was empty and we added some files to it, start uploading them
if (wasQueueEmpty && this.fileQueue.length > 0) {
this.dequeueNextFile();
}
}
dequeueNextFile() {
const file = this.fileQueue.shift();
// request an upload authentication token and then upload a file using said token
socket.emit("upload:auth");
socket.once("upload:auth", (token) => {
this.uploadSingleFile(file, token);
});
}
setProgress(value) {
this.uploadProgressbar.classList.toggle("upload-progressbar-visible", value > 0);
this.uploadProgressbar.style.width = value + "%";
}
uploadSingleFile(file, token) {
const xhr = new XMLHttpRequest();
this.xhr = xhr;
xhr.upload.addEventListener("progress", (e) => {
const percent = Math.floor(e.loaded / e.total * 1000) / 10;
this.setProgress(percent);
}, false);
xhr.onreadystatechange = () => {
if (xhr.readyState === XMLHttpRequest.DONE) {
this.xhr = null;
let response;
try {
response = JSON.parse(xhr.responseText);
} catch (err) {
// This is just a safe guard and should not happen if server doesn't throw any errors.
// Browsers break the HTTP spec by aborting the request without reading any response data,
// if there is still data to be uploaded. Servers will only error in extreme cases like bad
// authentication or server-side errors.
response = {
error: "Connection aborted",
};
}
this.handleResponse(response);
// this file was processed, if we still have files in the queue, upload the next one
if (this.fileQueue.length > 0) {
this.dequeueNextFile();
}
}
};
const formData = new FormData();
formData.append("file", file);
xhr.open("POST", `/uploads/new/${token}`);
xhr.send(formData);
}
handleResponse(response) {
this.setProgress(0);
if (response.error) {
// require here due to circular dependency
const {vueApp} = require("./vue");
vueApp.currentUserVisibleError = response.error;
return;
}
if (response.url) {
this.insertUploadUrl(response.url);
}
}
insertUploadUrl(url) {
const fullURL = (new URL(url, location)).toString();
const textbox = document.getElementById("input");
const initStart = textbox.selectionStart;
// Get the text before the cursor, and add a space if it's not in the beginning
const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : "";
// Get the remaining text after the cursor
const cursorToTail = textbox.value.substr(initStart);
// Construct the value until the point where we want the cursor to be
const textBeforeTail = headToCursor + fullURL + " ";
updateCursor(textbox, textBeforeTail + cursorToTail);
// Set the cursor after the link and a space
textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length;
}
// TODO: This is a temporary hack while Vue porting is finalized
abort() {
this.fileQueue = [];
if (this.xhr) {
this.xhr.abort();
this.xhr = null;
}
}
}
const instance = new Uploader();
function initialize() {
instance.listenOnInput(document.getElementById("upload-input"));
instance.listenOnDrop(document);
instance.addEventListener("complete", () => {
// Reset progressbar
$("#upload-progressbar").width(0);
});
instance.addEventListener("progress", (event) => {
const percent = `${((event.bytesLoaded / event.file.size) * 100)}%`;
$("#upload-progressbar").width(percent);
});
instance.addEventListener("error", (event) => {
// Reset progressbar
$("#upload-progressbar").width(0);
vueApp.currentUserVisibleError = event.message;
});
const $form = $(document);
const $overlay = $("#upload-overlay");
$form.on("dragover", () => {
$overlay.addClass("is-dragover");
return false;
});
$form.on("dragend dragleave drop", () => {
$overlay.removeClass("is-dragover");
return false;
});
instance.init();
return instance;
}
/**
@ -48,4 +208,8 @@ function setMaxFileSize(kb) {
instance.maxFileSize = kb;
}
module.exports = {initialize, setMaxFileSize};
module.exports = {
abort: () => instance.abort(),
initialize,
setMaxFileSize,
};

2
package.json
View File

@ -40,6 +40,7 @@
},
"dependencies": {
"bcryptjs": "2.4.3",
"busboy": "0.3.0",
"chalk": "2.4.2",
"cheerio": "0.22.0",
"commander": "2.19.0",
@ -58,7 +59,6 @@
"request": "2.88.0",
"semver": "5.6.0",
"socket.io": "2.2.0",
"socketio-file-upload": "0.7.0",
"thelounge-ldapjs-non-maintained-fork": "1.0.2",
"tlds": "1.203.1",
"ua-parser-js": "0.7.19",

227
src/plugins/uploader.js
View File

@ -1,7 +1,8 @@
"use strict";
const SocketIOFileUploadServer = require("socketio-file-upload/server");
const Helper = require("../helper");
const busboy = require("busboy");
const uuidv4 = require("uuid/v4");
const path = require("path");
const fsextra = require("fs-extra");
const fs = require("fs");
@ -28,51 +29,20 @@ const whitelist = [
"video/webm",
];
const uploadTokens = new Map();
class Uploader {
constructor(socket) {
const uploader = new SocketIOFileUploadServer();
const folder = path.join(Helper.getFileUploadPath(), ".tmp");
socket.on("upload:auth", () => {
const token = uuidv4();
fsextra.ensureDir(folder, (err) => {
if (err) {
log.err(`Error ensuring ${folder} exists for uploads.`);
} else {
uploader.dir = folder;
}
uploadTokens.set(token, true);
socket.emit("upload:auth", token);
// Invalidate the token in one minute
setTimeout(() => uploadTokens.delete(token), 60 * 1000);
});
uploader.on("complete", (data) => {
let randomName;
let destPath;
// Generate a random file name for storage on disk
do {
randomName = crypto.randomBytes(8).toString("hex");
destPath = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2), randomName);
} while (fs.existsSync(destPath));
fsextra.move(data.file.pathName, destPath).then(() => {
const slug = encodeURIComponent(path.basename(data.file.pathName));
const url = `uploads/${randomName}/${slug}`;
socket.emit("upload:success", url);
}).catch(() => {
log.warn(`Unable to move uploaded file "${data.file.pathName}"`);
// Emit failed upload to the client if file move fails
socket.emit("siofu_error", {
id: data.file.id,
message: "Unable to move uploaded file",
});
});
});
uploader.on("error", (data) => {
log.error(`File upload failed: ${data.error}`);
});
// maxFileSize is in bytes, but config option is passed in as KB
uploader.maxFileSize = Uploader.getMaxFileSize();
uploader.listen(socket);
}
static isValidType(mimeType) {
@ -80,43 +50,173 @@ class Uploader {
}
static router(express) {
express.get("/uploads/:name/:slug*?", (req, res) => {
const name = req.params.name;
express.get("/uploads/:name/:slug*?", Uploader.routeGetFile);
express.post("/uploads/new/:token", Uploader.routeUploadFile);
}
const nameRegex = /^[0-9a-f]{16}$/;
static routeGetFile(req, res) {
const name = req.params.name;
if (!nameRegex.test(name)) {
return res.status(404).send("Not found");
const nameRegex = /^[0-9a-f]{16}$/;
if (!nameRegex.test(name)) {
return res.status(404).send("Not found");
}
const folder = name.substring(0, 2);
const uploadPath = Helper.getFileUploadPath();
const filePath = path.join(uploadPath, folder, name);
const detectedMimeType = Uploader.getFileType(filePath);
// doesn't exist
if (detectedMimeType === null) {
return res.status(404).send("Not found");
}
// Force a download in the browser if it's not a whitelisted type (binary or otherwise unknown)
const contentDisposition = Uploader.isValidType(detectedMimeType) ? "inline" : "attachment";
res.setHeader("Content-Disposition", contentDisposition);
res.setHeader("Cache-Control", "max-age=86400");
res.contentType(detectedMimeType);
return res.sendFile(filePath);
}
static routeUploadFile(req, res) {
let busboyInstance;
let uploadUrl;
let randomName;
let destDir;
let destPath;
let streamWriter;
const doneCallback = () => {
// detach the stream and drain any remaining data
if (busboyInstance) {
req.unpipe(busboyInstance);
req.on("readable", req.read.bind(req));
busboyInstance.removeAllListeners();
busboyInstance = null;
}
const folder = name.substring(0, 2);
const uploadPath = Helper.getFileUploadPath();
const filePath = path.join(uploadPath, folder, name);
const detectedMimeType = Uploader.getFileType(filePath);
// close the output file stream
if (streamWriter) {
streamWriter.end();
streamWriter = null;
}
};
// doesn't exist
if (detectedMimeType === null) {
return res.status(404).send("Not found");
const abortWithError = (err) => {
doneCallback();
// if we ended up erroring out, delete the output file from disk
if (destPath && fs.existsSync(destPath)) {
fs.unlinkSync(destPath);
destPath = null;
}
// Force a download in the browser if it's not a whitelisted type (binary or otherwise unknown)
const contentDisposition = Uploader.isValidType(detectedMimeType) ? "inline" : "attachment";
return res.status(400).json({error: err.message});
};
res.setHeader("Content-Disposition", contentDisposition);
res.setHeader("Cache-Control", "max-age=86400");
res.contentType(detectedMimeType);
// if the authentication token is incorrect, bail out
if (uploadTokens.delete(req.params.token) !== true) {
return abortWithError(Error("Unauthorized"));
}
return res.sendFile(filePath);
// if the request does not contain any body data, bail out
if (req.headers["content-length"] < 1) {
return abortWithError(Error("Length Required"));
}
// Only allow multipart, as busboy can throw an error on unsupported types
if (!req.headers["content-type"].startsWith("multipart/form-data")) {
return abortWithError(Error("Unsupported Content Type"));
}
// create a new busboy processor, it is wrapped in try/catch
// because it can throw on malformed headers
try {
busboyInstance = new busboy({
headers: req.headers,
limits: {
files: 1, // only allow one file per upload
fileSize: Uploader.getMaxFileSize(),
},
});
} catch (err) {
return abortWithError(err);
}
// Any error or limit from busboy will abort the upload with an error
busboyInstance.on("error", abortWithError);
busboyInstance.on("partsLimit", () => abortWithError(Error("Parts limit reached")));
busboyInstance.on("filesLimit", () => abortWithError(Error("Files limit reached")));
busboyInstance.on("fieldsLimit", () => abortWithError(Error("Fields limit reached")));
// generate a random output filename for the file
// we use do/while loop to prevent the rare case of generating a file name
// that already exists on disk
do {
randomName = crypto.randomBytes(8).toString("hex");
destDir = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2));
destPath = path.join(destDir, randomName);
} while (fs.existsSync(destPath));
// we split the filename into subdirectories (by taking 2 letters from the beginning)
// this helps avoid file system and certain tooling limitations when there are
// too many files on one folder
try {
fsextra.ensureDirSync(destDir);
} catch (err) {
log.err(`Error ensuring ${destDir} exists for uploads: ${err.message}`);
return abortWithError(err);
}
// Open a file stream for writing
streamWriter = fs.createWriteStream(destPath);
streamWriter.on("error", abortWithError);
busboyInstance.on("file", (fieldname, fileStream, filename) => {
uploadUrl = `uploads/${randomName}/${encodeURIComponent(filename)}`;
// if the busboy data stream errors out or goes over the file size limit
// abort the processing with an error
fileStream.on("error", abortWithError);
fileStream.on("limit", () => {
fileStream.unpipe(streamWriter);
fileStream.on("readable", fileStream.read.bind(fileStream));
abortWithError(Error("File size limit reached"));
});
// Attempt to write the stream to file
fileStream.pipe(streamWriter);
});
busboyInstance.on("finish", () => {
doneCallback();
// upload was done, send the generated file url to the client
res.status(200).json({
url: uploadUrl,
});
});
// pipe request body to busboy for processing
return req.pipe(busboyInstance);
}
static getMaxFileSize() {
const configOption = Helper.config.fileUpload.maxFileSize;
if (configOption === -1) { // no file size limit
return null;
// Busboy uses Infinity to allow unlimited file size
if (configOption < 1) {
return Infinity;
}
// maxFileSize is in bytes, but config option is passed in as KB
return configOption * 1024;
}
@ -129,14 +229,17 @@ class Uploader {
// returns {ext, mime} if found, null if not.
const file = fileType(buffer);
// if a file type was detected correctly, return it
if (file) {
return file.mime;
}
// if the buffer is a valid UTF-8 buffer, use text/plain
if (isUtf8(buffer)) {
return "text/plain";
}
// otherwise assume it's random binary data
return "application/octet-stream";
} catch (e) {
if (e.code !== "ENOENT") {

24
yarn.lock
View File

@ -1641,6 +1641,13 @@ builtin-status-codes@^3.0.0:
resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8"
integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
busboy@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.0.tgz#6ee3cb1c844fc1f691d8f9d824f70128b3b5e485"
integrity sha512-e+kzZRAbbvJPLjQz2z+zAyr78BSi9IFeBTyLwF76g78Q2zRt/RZ1NtS3MS17v2yLqYfLz69zHdC+1L4ja8PwqQ==
dependencies:
dicer "0.3.0"
bytes@3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
@ -2694,6 +2701,13 @@ detect-node@^2.0.4:
resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c"
integrity sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==
dicer@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.3.0.tgz#eacd98b3bfbf92e8ab5c2fdb71aaac44bb06b872"
integrity sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==
dependencies:
streamsearch "0.1.2"
diff@3.5.0, diff@^3.5.0:
version "3.5.0"
resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
@ -7711,11 +7725,6 @@ socket.io@2.2.0:
socket.io-client "2.2.0"
socket.io-parser "~3.3.0"
socketio-file-upload@0.7.0:
version "0.7.0"
resolved "https://registry.yarnpkg.com/socketio-file-upload/-/socketio-file-upload-0.7.0.tgz#aec62bb20051cbc6019792ff4dcc3907101eb466"
integrity sha512-eKL6LAPiDH9xIsn2eP0ZDzw/EUsnDiZxuDN6TpyPcsaS42Q5wt5VmeN6CGcPsMx57UMQ6jxjIljoBTnCVsG8Qw==
sockjs-client@1.3.0:
version "1.3.0"
resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.3.0.tgz#12fc9d6cb663da5739d3dc5fb6e8687da95cb177"
@ -7963,6 +7972,11 @@ stream-shift@^1.0.0:
resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
streamsearch@0.1.2:
version "0.1.2"
resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
strict-uri-encode@^1.0.0:
version "1.1.0"
resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"