Merge pull request #3037 from thelounge/xpaw-richrd/file-uploads

Re-implement file uploading with old school multipart forms

Co-authored-by: Jérémie Astori <jeremie@astori.fr>

client/components/ChatInput.vue

@@ -153,6 +153,7 @@ export default {
 	},
 	destroyed() {
 		require("../js/autocompletion").disable();
+		upload.abort();
 	},
 	methods: {
 		setPendingMessage(e) {

client/css/style.css

@@ -30,6 +30,9 @@
 	/* Background and left-border color of highlight messages */
 	--highlight-bg-color: #efe8dc;
 	--highlight-border-color: #b08c4f;
+
+	/* Color of the progress bar that appears as a file is being uploaded to the server. Defaults to button color */
+	--upload-progressbar-color: var(--button-color);
 }
 
 /* Samsung Internet <7.0 and Microsoft Edge support (yes, both of them use webkit prefix) */
@@ -1956,9 +1959,16 @@ part/quit messages where we don't load previews (adds a blank line otherwise) */
 }
 
 #upload-progressbar {
-	background: blue;
+	background: var(--upload-progressbar-color);
+	box-shadow: 0 0 10px var(--upload-progressbar-color);
 	width: 0%;
 	height: 2px;
+	visibility: hidden;
+}
+
+#upload-progressbar.upload-progressbar-visible {
+	visibility: visible;
+	transition: 0.3s width ease-in-out;
 }
 
 #form {

client/js/socket-events/index.js

@@ -16,7 +16,6 @@ require("./part");
 require("./quit");
 require("./sync_sort");
 require("./topic");
-require("./uploads");
 require("./users");
 require("./sign_out");
 require("./sessions_list");

client/js/socket-events/uploads.js

@@ -1,24 +0,0 @@
-"use strict";
-
-const socket = require("../socket");
-const updateCursor = require("undate").update;
-
-socket.on("upload:success", (url) => {
-	const fullURL = (new URL(url, location)).toString();
-	const textbox = document.getElementById("input");
-	const initStart = textbox.selectionStart;
-
-	// Get the text before the cursor, and add a space if it's not in the beginning
-	const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : "";
-
-	// Get the remaining text after the cursor
-	const cursorToTail = textbox.value.substr(initStart);
-
-	// Construct the value until the point where we want the cursor to be
-	const textBeforeTail = headToCursor + fullURL + " ";
-
-	updateCursor(textbox, textBeforeTail + cursorToTail);
-
-	// Set the cursor after the link and a space
-	textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length;
-});

client/js/upload.js

@@ -1,43 +1,203 @@
 "use strict";
 
-const $ = require("jquery");
 const socket = require("./socket");
-const SocketIOFileUpload = require("socketio-file-upload/client");
+const updateCursor = require("undate").update;
-const instance = new SocketIOFileUpload(socket);
+
+class Uploader {
+	init() {
+		this.xhr = null;
+		this.fileQueue = [];
+		this.overlay = document.getElementById("upload-overlay");
+		this.uploadInput = document.getElementById("upload-input");
+		this.uploadProgressbar = document.getElementById("upload-progressbar");
+
+		this.uploadInput.addEventListener("change", (e) => this.filesChanged(e));
+		document.addEventListener("dragenter", (e) => this.dragEnter(e));
+		document.addEventListener("dragover", (e) => this.dragOver(e));
+		document.addEventListener("dragleave", (e) => this.dragLeave(e));
+		document.addEventListener("drop", (e) => this.drop(e));
+	}
+
+	dragOver(event) {
+		// Prevent dragover event completely and do nothing with it
+		// This stops the browser from trying to guess which cursor to show
+		event.preventDefault();
+	}
+
+	dragEnter(event) {
+		event.preventDefault();
+
+		// relatedTarget is the target where we entered the drag from
+		// when dragging from another window, the target is null, otherwise its a DOM element
+		if (!event.relatedTarget && event.dataTransfer.types.includes("Files")) {
+			this.overlay.classList.add("is-dragover");
+		}
+	}
+
+	dragLeave(event) {
+		event.preventDefault();
+
+		// If relatedTarget is null, that means we are no longer dragging over the page
+		if (!event.relatedTarget) {
+			this.overlay.classList.remove("is-dragover");
+		}
+	}
+
+	drop(event) {
+		event.preventDefault();
+		this.overlay.classList.remove("is-dragover");
+
+		let files;
+
+		if (event.dataTransfer.items) {
+			files = Array.from(event.dataTransfer.items)
+				.filter((item) => item.kind === "file")
+				.map((item) => item.getAsFile());
+		} else {
+			files = Array.from(event.dataTransfer.files);
+		}
+
+		this.triggerUpload(files);
+	}
+
+	filesChanged() {
+		const files = Array.from(this.uploadInput.files);
+		this.triggerUpload(files);
+	}
+
+	triggerUpload(files) {
+		if (!files.length) {
+			return;
+		}
+
+		const wasQueueEmpty = this.fileQueue.length === 0;
+
+		for (const file of files) {
+			if (this.maxFileSize > 0 && file.size > this.maxFileSize) {
+				this.handleResponse({
+					error: `File ${file.name} is over the maximum allowed size`,
+				});
+
+				continue;
+			}
+
+			this.fileQueue.push(file);
+		}
+
+		// if the queue was empty and we added some files to it, start uploading them
+		if (wasQueueEmpty && this.fileQueue.length > 0) {
+			this.dequeueNextFile();
+		}
+	}
+
+	dequeueNextFile() {
+		const file = this.fileQueue.shift();
+
+		// request an upload authentication token and then upload a file using said token
+		socket.emit("upload:auth");
+		socket.once("upload:auth", (token) => {
+			this.uploadSingleFile(file, token);
+		});
+	}
+
+	setProgress(value) {
+		this.uploadProgressbar.classList.toggle("upload-progressbar-visible", value > 0);
+		this.uploadProgressbar.style.width = value + "%";
+	}
+
+	uploadSingleFile(file, token) {
+		const xhr = new XMLHttpRequest();
+		this.xhr = xhr;
+
+		xhr.upload.addEventListener("progress", (e) => {
+			const percent = Math.floor(e.loaded / e.total * 1000) / 10;
+			this.setProgress(percent);
+		}, false);
+
+		xhr.onreadystatechange = () => {
+			if (xhr.readyState === XMLHttpRequest.DONE) {
+				this.xhr = null;
+
+				let response;
+
+				try {
+					response = JSON.parse(xhr.responseText);
+				} catch (err) {
+					// This is just a safe guard and should not happen if server doesn't throw any errors.
+					// Browsers break the HTTP spec by aborting the request without reading any response data,
+					// if there is still data to be uploaded. Servers will only error in extreme cases like bad
+					// authentication or server-side errors.
+					response = {
+						error: "Connection aborted",
+					};
+				}
+
+				this.handleResponse(response);
+
+				// this file was processed, if we still have files in the queue, upload the next one
+				if (this.fileQueue.length > 0) {
+					this.dequeueNextFile();
+				}
+			}
+		};
+
+		const formData = new FormData();
+		formData.append("file", file);
+		xhr.open("POST", `/uploads/new/${token}`);
+		xhr.send(formData);
+	}
+
+	handleResponse(response) {
+		this.setProgress(0);
+
+		if (response.error) {
+			// require here due to circular dependency
 			const {vueApp} = require("./vue");
+			vueApp.currentUserVisibleError = response.error;
+			return;
+		}
+
+		if (response.url) {
+			this.insertUploadUrl(response.url);
+		}
+	}
+
+	insertUploadUrl(url) {
+		const fullURL = (new URL(url, location)).toString();
+		const textbox = document.getElementById("input");
+		const initStart = textbox.selectionStart;
+
+		// Get the text before the cursor, and add a space if it's not in the beginning
+		const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : "";
+
+		// Get the remaining text after the cursor
+		const cursorToTail = textbox.value.substr(initStart);
+
+		// Construct the value until the point where we want the cursor to be
+		const textBeforeTail = headToCursor + fullURL + " ";
+
+		updateCursor(textbox, textBeforeTail + cursorToTail);
+
+		// Set the cursor after the link and a space
+		textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length;
+	}
+
+	// TODO: This is a temporary hack while Vue porting is finalized
+	abort() {
+		this.fileQueue = [];
+
+		if (this.xhr) {
+			this.xhr.abort();
+			this.xhr = null;
+		}
+	}
+}
+
+const instance = new Uploader();
 
 function initialize() {
-	instance.listenOnInput(document.getElementById("upload-input"));
+	instance.init();
-	instance.listenOnDrop(document);
+	return instance;
-
-	instance.addEventListener("complete", () => {
-		// Reset progressbar
-		$("#upload-progressbar").width(0);
-	});
-
-	instance.addEventListener("progress", (event) => {
-		const percent = `${((event.bytesLoaded / event.file.size) * 100)}%`;
-		$("#upload-progressbar").width(percent);
-	});
-
-	instance.addEventListener("error", (event) => {
-		// Reset progressbar
-		$("#upload-progressbar").width(0);
-		vueApp.currentUserVisibleError = event.message;
-	});
-
-	const $form = $(document);
-	const $overlay = $("#upload-overlay");
-
-	$form.on("dragover", () => {
-		$overlay.addClass("is-dragover");
-		return false;
-	});
-
-	$form.on("dragend dragleave drop", () => {
-		$overlay.removeClass("is-dragover");
-		return false;
-	});
 }
 
 /**
@@ -48,4 +208,8 @@ function setMaxFileSize(kb) {
 	instance.maxFileSize = kb;
 }
 
-module.exports = {initialize, setMaxFileSize};
+module.exports = {
+	abort: () => instance.abort(),
+	initialize,
+	setMaxFileSize,
+};

package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "bcryptjs": "2.4.3",
+    "busboy": "0.3.0",
     "chalk": "2.4.2",
     "cheerio": "0.22.0",
     "commander": "2.19.0",
@@ -58,7 +59,6 @@
     "request": "2.88.0",
     "semver": "5.6.0",
     "socket.io": "2.2.0",
-    "socketio-file-upload": "0.7.0",
     "thelounge-ldapjs-non-maintained-fork": "1.0.2",
     "tlds": "1.203.1",
     "ua-parser-js": "0.7.19",

src/plugins/uploader.js

@@ -1,7 +1,8 @@
 "use strict";
 
-const SocketIOFileUploadServer = require("socketio-file-upload/server");
 const Helper = require("../helper");
+const busboy = require("busboy");
+const uuidv4 = require("uuid/v4");
 const path = require("path");
 const fsextra = require("fs-extra");
 const fs = require("fs");
@@ -28,51 +29,20 @@ const whitelist = [
 	"video/webm",
 ];
 
+const uploadTokens = new Map();
+
 class Uploader {
 	constructor(socket) {
-		const uploader = new SocketIOFileUploadServer();
+		socket.on("upload:auth", () => {
-		const folder = path.join(Helper.getFileUploadPath(), ".tmp");
+			const token = uuidv4();
 
-		fsextra.ensureDir(folder, (err) => {
+			uploadTokens.set(token, true);
-			if (err) {
+
-				log.err(`Error ensuring ${folder} exists for uploads.`);
+			socket.emit("upload:auth", token);
-			} else {
+
-				uploader.dir = folder;
+			// Invalidate the token in one minute
-			}
+			setTimeout(() => uploadTokens.delete(token), 60 * 1000);
 		});
-
-		uploader.on("complete", (data) => {
-			let randomName;
-			let destPath;
-
-			// Generate a random file name for storage on disk
-			do {
-				randomName = crypto.randomBytes(8).toString("hex");
-				destPath = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2), randomName);
-			} while (fs.existsSync(destPath));
-
-			fsextra.move(data.file.pathName, destPath).then(() => {
-				const slug = encodeURIComponent(path.basename(data.file.pathName));
-				const url = `uploads/${randomName}/${slug}`;
-				socket.emit("upload:success", url);
-			}).catch(() => {
-				log.warn(`Unable to move uploaded file "${data.file.pathName}"`);
-
-				// Emit failed upload to the client if file move fails
-				socket.emit("siofu_error", {
-					id: data.file.id,
-					message: "Unable to move uploaded file",
-				});
-			});
-		});
-
-		uploader.on("error", (data) => {
-			log.error(`File upload failed: ${data.error}`);
-		});
-
-		// maxFileSize is in bytes, but config option is passed in as KB
-		uploader.maxFileSize = Uploader.getMaxFileSize();
-		uploader.listen(socket);
 	}
 
 	static isValidType(mimeType) {
@@ -80,7 +50,11 @@ class Uploader {
 	}
 
 	static router(express) {
-		express.get("/uploads/:name/:slug*?", (req, res) => {
+		express.get("/uploads/:name/:slug*?", Uploader.routeGetFile);
+		express.post("/uploads/new/:token", Uploader.routeUploadFile);
+	}
+
+	static routeGetFile(req, res) {
 		const name = req.params.name;
 
 		const nameRegex = /^[0-9a-f]{16}$/;
@@ -107,16 +81,142 @@ class Uploader {
 		res.contentType(detectedMimeType);
 
 		return res.sendFile(filePath);
+	}
+
+	static routeUploadFile(req, res) {
+		let busboyInstance;
+		let uploadUrl;
+		let randomName;
+		let destDir;
+		let destPath;
+		let streamWriter;
+
+		const doneCallback = () => {
+			// detach the stream and drain any remaining data
+			if (busboyInstance) {
+				req.unpipe(busboyInstance);
+				req.on("readable", req.read.bind(req));
+
+				busboyInstance.removeAllListeners();
+				busboyInstance = null;
+			}
+
+			// close the output file stream
+			if (streamWriter) {
+				streamWriter.end();
+				streamWriter = null;
+			}
+		};
+
+		const abortWithError = (err) => {
+			doneCallback();
+
+			// if we ended up erroring out, delete the output file from disk
+			if (destPath && fs.existsSync(destPath)) {
+				fs.unlinkSync(destPath);
+				destPath = null;
+			}
+
+			return res.status(400).json({error: err.message});
+		};
+
+		// if the authentication token is incorrect, bail out
+		if (uploadTokens.delete(req.params.token) !== true) {
+			return abortWithError(Error("Unauthorized"));
+		}
+
+		// if the request does not contain any body data, bail out
+		if (req.headers["content-length"] < 1) {
+			return abortWithError(Error("Length Required"));
+		}
+
+		// Only allow multipart, as busboy can throw an error on unsupported types
+		if (!req.headers["content-type"].startsWith("multipart/form-data")) {
+			return abortWithError(Error("Unsupported Content Type"));
+		}
+
+		// create a new busboy processor, it is wrapped in try/catch
+		// because it can throw on malformed headers
+		try {
+			busboyInstance = new busboy({
+				headers: req.headers,
+				limits: {
+					files: 1, // only allow one file per upload
+					fileSize: Uploader.getMaxFileSize(),
+				},
 			});
+		} catch (err) {
+			return abortWithError(err);
+		}
+
+		// Any error or limit from busboy will abort the upload with an error
+		busboyInstance.on("error", abortWithError);
+		busboyInstance.on("partsLimit", () => abortWithError(Error("Parts limit reached")));
+		busboyInstance.on("filesLimit", () => abortWithError(Error("Files limit reached")));
+		busboyInstance.on("fieldsLimit", () => abortWithError(Error("Fields limit reached")));
+
+		// generate a random output filename for the file
+		// we use do/while loop to prevent the rare case of generating a file name
+		// that already exists on disk
+		do {
+			randomName = crypto.randomBytes(8).toString("hex");
+			destDir = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2));
+			destPath = path.join(destDir, randomName);
+		} while (fs.existsSync(destPath));
+
+		// we split the filename into subdirectories (by taking 2 letters from the beginning)
+		// this helps avoid file system and certain tooling limitations when there are
+		// too many files on one folder
+		try {
+			fsextra.ensureDirSync(destDir);
+		} catch (err) {
+			log.err(`Error ensuring ${destDir} exists for uploads: ${err.message}`);
+			return abortWithError(err);
+		}
+
+		// Open a file stream for writing
+		streamWriter = fs.createWriteStream(destPath);
+		streamWriter.on("error", abortWithError);
+
+		busboyInstance.on("file", (fieldname, fileStream, filename) => {
+			uploadUrl = `uploads/${randomName}/${encodeURIComponent(filename)}`;
+
+			// if the busboy data stream errors out or goes over the file size limit
+			// abort the processing with an error
+			fileStream.on("error", abortWithError);
+			fileStream.on("limit", () => {
+				fileStream.unpipe(streamWriter);
+				fileStream.on("readable", fileStream.read.bind(fileStream));
+
+				abortWithError(Error("File size limit reached"));
+			});
+
+			// Attempt to write the stream to file
+			fileStream.pipe(streamWriter);
+		});
+
+		busboyInstance.on("finish", () => {
+			doneCallback();
+
+			// upload was done, send the generated file url to the client
+			res.status(200).json({
+				url: uploadUrl,
+			});
+		});
+
+		// pipe request body to busboy for processing
+		return req.pipe(busboyInstance);
 	}
 
 	static getMaxFileSize() {
 		const configOption = Helper.config.fileUpload.maxFileSize;
 
-		if (configOption === -1) { // no file size limit
+		// Busboy uses Infinity to allow unlimited file size
-			return null;
+		if (configOption < 1) {
+			return Infinity;
 		}
 
+		// maxFileSize is in bytes, but config option is passed in as KB
 		return configOption * 1024;
 	}
 
@@ -129,14 +229,17 @@ class Uploader {
 			// returns {ext, mime} if found, null if not.
 			const file = fileType(buffer);
 
+			// if a file type was detected correctly, return it
 			if (file) {
 				return file.mime;
 			}
 
+			// if the buffer is a valid UTF-8 buffer, use text/plain
 			if (isUtf8(buffer)) {
 				return "text/plain";
 			}
 
+			// otherwise assume it's random binary data
 			return "application/octet-stream";
 		} catch (e) {
 			if (e.code !== "ENOENT") {

yarn.lock

@@ -1641,6 +1641,13 @@ builtin-status-codes@^3.0.0:
   resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8"
   integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=
 
+busboy@0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.0.tgz#6ee3cb1c844fc1f691d8f9d824f70128b3b5e485"
+  integrity sha512-e+kzZRAbbvJPLjQz2z+zAyr78BSi9IFeBTyLwF76g78Q2zRt/RZ1NtS3MS17v2yLqYfLz69zHdC+1L4ja8PwqQ==
+  dependencies:
+    dicer "0.3.0"
+
 bytes@3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048"
@@ -2694,6 +2701,13 @@ detect-node@^2.0.4:
   resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c"
   integrity sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw==
 
+dicer@0.3.0:
+  version "0.3.0"
+  resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.3.0.tgz#eacd98b3bfbf92e8ab5c2fdb71aaac44bb06b872"
+  integrity sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA==
+  dependencies:
+    streamsearch "0.1.2"
+
 diff@3.5.0, diff@^3.5.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12"
@@ -7711,11 +7725,6 @@ socket.io@2.2.0:
     socket.io-client "2.2.0"
     socket.io-parser "~3.3.0"
 
-socketio-file-upload@0.7.0:
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/socketio-file-upload/-/socketio-file-upload-0.7.0.tgz#aec62bb20051cbc6019792ff4dcc3907101eb466"
-  integrity sha512-eKL6LAPiDH9xIsn2eP0ZDzw/EUsnDiZxuDN6TpyPcsaS42Q5wt5VmeN6CGcPsMx57UMQ6jxjIljoBTnCVsG8Qw==
-
 sockjs-client@1.3.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.3.0.tgz#12fc9d6cb663da5739d3dc5fb6e8687da95cb177"
@@ -7963,6 +7972,11 @@ stream-shift@^1.0.0:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
   integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
 
+streamsearch@0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"
+  integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=
+
 strict-uri-encode@^1.0.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"