From f84e4199e9a6fe9406f081643371803b57b4d0d4 Mon Sep 17 00:00:00 2001 From: Pavel Djundik Date: Tue, 19 Feb 2019 17:12:08 +0200 Subject: [PATCH] Re-implement file uploads with old-school forms Co-Authored-By: Richard Lewis --- client/css/style.css | 6 + client/js/socket-events/index.js | 1 - client/js/socket-events/uploads.js | 24 --- client/js/upload.js | 216 ++++++++++++++++++++++----- package.json | 2 +- src/plugins/uploader.js | 227 +++++++++++++++++++++-------- yarn.lock | 24 ++- 7 files changed, 372 insertions(+), 128 deletions(-) delete mode 100644 client/js/socket-events/uploads.js diff --git a/client/css/style.css b/client/css/style.css index 0e5854df..c570e53c 100644 --- a/client/css/style.css +++ b/client/css/style.css @@ -1959,6 +1959,12 @@ part/quit messages where we don't load previews (adds a blank line otherwise) */ background: blue; width: 0%; height: 2px; + visibility: hidden; +} + +#upload-progressbar.upload-progressbar-visible { + visibility: visible; + transition: 0.3s width ease-in-out; } #form { diff --git a/client/js/socket-events/index.js b/client/js/socket-events/index.js index 78ea469a..8bc1d5a8 100644 --- a/client/js/socket-events/index.js +++ b/client/js/socket-events/index.js @@ -16,7 +16,6 @@ require("./part"); require("./quit"); require("./sync_sort"); require("./topic"); -require("./uploads"); require("./users"); require("./sign_out"); require("./sessions_list"); diff --git a/client/js/socket-events/uploads.js b/client/js/socket-events/uploads.js deleted file mode 100644 index c6f5bebf..00000000 --- a/client/js/socket-events/uploads.js +++ /dev/null @@ -1,24 +0,0 @@ -"use strict"; - -const socket = require("../socket"); -const updateCursor = require("undate").update; - -socket.on("upload:success", (url) => { - const fullURL = (new URL(url, location)).toString(); - const textbox = document.getElementById("input"); - const initStart = textbox.selectionStart; - - // Get the text before the cursor, and add a space if it's not in the beginning - const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : ""; - - // Get the remaining text after the cursor - const cursorToTail = textbox.value.substr(initStart); - - // Construct the value until the point where we want the cursor to be - const textBeforeTail = headToCursor + fullURL + " "; - - updateCursor(textbox, textBeforeTail + cursorToTail); - - // Set the cursor after the link and a space - textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length; -}); diff --git a/client/js/upload.js b/client/js/upload.js index 361f6665..0feb5756 100644 --- a/client/js/upload.js +++ b/client/js/upload.js @@ -1,43 +1,189 @@ "use strict"; -const $ = require("jquery"); const socket = require("./socket"); -const SocketIOFileUpload = require("socketio-file-upload/client"); -const instance = new SocketIOFileUpload(socket); -const {vueApp} = require("./vue"); +const updateCursor = require("undate").update; + +class Uploader { + init() { + this.fileQueue = []; + this.overlay = document.getElementById("upload-overlay"); + this.uploadInput = document.getElementById("upload-input"); + this.uploadProgressbar = document.getElementById("upload-progressbar"); + + this.uploadInput.addEventListener("change", (e) => this.filesChanged(e)); + document.addEventListener("dragenter", (e) => this.dragEnter(e)); + document.addEventListener("dragover", (e) => this.dragOver(e)); + document.addEventListener("dragleave", (e) => this.dragLeave(e)); + document.addEventListener("drop", (e) => this.drop(e)); + } + + dragOver(event) { + // Prevent dragover event completely and do nothing with it + // This stops the browser from trying to guess which cursor to show + event.preventDefault(); + } + + dragEnter(event) { + event.preventDefault(); + + // relatedTarget is the target where we entered the drag from + // when dragging from another window, the target is null, otherwise its a DOM element + if (!event.relatedTarget && event.dataTransfer.types.includes("Files")) { + this.overlay.classList.add("is-dragover"); + } + } + + dragLeave(event) { + event.preventDefault(); + + // If relatedTarget is null, that means we are no longer dragging over the page + if (!event.relatedTarget) { + this.overlay.classList.remove("is-dragover"); + } + } + + drop(event) { + event.preventDefault(); + this.overlay.classList.remove("is-dragover"); + + let files; + + if (event.dataTransfer.items) { + files = Array.from(event.dataTransfer.items) + .filter((item) => item.kind === "file") + .map((item) => item.getAsFile()); + } else { + files = Array.from(event.dataTransfer.files); + } + + this.triggerUpload(files); + } + + filesChanged() { + const files = Array.from(this.uploadInput.files); + this.triggerUpload(files); + } + + triggerUpload(files) { + if (!files.length) { + return; + } + + const wasQueueEmpty = this.fileQueue.length === 0; + + for (const file of files) { + if (this.maxFileSize > 0 && file.size > this.maxFileSize) { + this.handleResponse({ + error: `File ${file.name} is over the maximum allowed size`, + }); + + continue; + } + + this.fileQueue.push(file); + } + + // if the queue was empty and we added some files to it, start uploading them + if (wasQueueEmpty && this.fileQueue.length > 0) { + this.dequeueNextFile(); + } + } + + dequeueNextFile() { + const file = this.fileQueue.shift(); + + // request an upload authentication token and then upload a file using said token + socket.emit("upload:auth"); + socket.once("upload:auth", (token) => { + this.uploadSingleFile(file, token); + }); + } + + setProgress(value) { + this.uploadProgressbar.classList.toggle("upload-progressbar-visible", value > 0); + this.uploadProgressbar.style.width = value + "%"; + } + + uploadSingleFile(file, token) { + const xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener("progress", (e) => { + const percent = Math.floor(e.loaded / e.total * 1000) / 10; + this.setProgress(percent); + }, false); + + xhr.onreadystatechange = () => { + if (xhr.readyState === XMLHttpRequest.DONE) { + let response; + + try { + response = JSON.parse(xhr.responseText); + } catch (err) { + // This is just a safe guard and should not happen if server doesn't throw any errors. + // Browsers break the HTTP spec by aborting the request without reading any response data, + // if there is still data to be uploaded. Servers will only error in extreme cases like bad + // authentication or server-side errors. + response = { + error: "Connection aborted", + }; + } + + this.handleResponse(response); + + // this file was processed, if we still have files in the queue, upload the next one + if (this.fileQueue.length > 0) { + this.dequeueNextFile(); + } + } + }; + + const formData = new FormData(); + formData.append("file", file); + xhr.open("POST", `/uploads/new/${token}`); + xhr.send(formData); + } + + handleResponse(response) { + this.setProgress(0); + + if (response.error) { + // require here due to circular dependency + const {vueApp} = require("./vue"); + vueApp.currentUserVisibleError = response.error; + return; + } + + if (response.url) { + this.insertUploadUrl(response.url); + } + } + + insertUploadUrl(url) { + const fullURL = (new URL(url, location)).toString(); + const textbox = document.getElementById("input"); + const initStart = textbox.selectionStart; + + // Get the text before the cursor, and add a space if it's not in the beginning + const headToCursor = initStart > 0 ? (textbox.value.substr(0, initStart) + " ") : ""; + + // Get the remaining text after the cursor + const cursorToTail = textbox.value.substr(initStart); + + // Construct the value until the point where we want the cursor to be + const textBeforeTail = headToCursor + fullURL + " "; + + updateCursor(textbox, textBeforeTail + cursorToTail); + + // Set the cursor after the link and a space + textbox.selectionStart = textbox.selectionEnd = textBeforeTail.length; + } +} + +const instance = new Uploader(); function initialize() { - instance.listenOnInput(document.getElementById("upload-input")); - instance.listenOnDrop(document); - - instance.addEventListener("complete", () => { - // Reset progressbar - $("#upload-progressbar").width(0); - }); - - instance.addEventListener("progress", (event) => { - const percent = `${((event.bytesLoaded / event.file.size) * 100)}%`; - $("#upload-progressbar").width(percent); - }); - - instance.addEventListener("error", (event) => { - // Reset progressbar - $("#upload-progressbar").width(0); - vueApp.currentUserVisibleError = event.message; - }); - - const $form = $(document); - const $overlay = $("#upload-overlay"); - - $form.on("dragover", () => { - $overlay.addClass("is-dragover"); - return false; - }); - - $form.on("dragend dragleave drop", () => { - $overlay.removeClass("is-dragover"); - return false; - }); + instance.init(); + return instance; } /** diff --git a/package.json b/package.json index adef8487..8019eb93 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "bcryptjs": "2.4.3", + "busboy": "0.3.0", "chalk": "2.4.2", "cheerio": "0.22.0", "commander": "2.19.0", @@ -58,7 +59,6 @@ "request": "2.88.0", "semver": "5.6.0", "socket.io": "2.2.0", - "socketio-file-upload": "0.7.0", "thelounge-ldapjs-non-maintained-fork": "1.0.2", "tlds": "1.203.1", "ua-parser-js": "0.7.19", diff --git a/src/plugins/uploader.js b/src/plugins/uploader.js index 7eaec02a..f7d19870 100644 --- a/src/plugins/uploader.js +++ b/src/plugins/uploader.js @@ -1,7 +1,8 @@ "use strict"; -const SocketIOFileUploadServer = require("socketio-file-upload/server"); const Helper = require("../helper"); +const busboy = require("busboy"); +const uuidv4 = require("uuid/v4"); const path = require("path"); const fsextra = require("fs-extra"); const fs = require("fs"); @@ -28,51 +29,20 @@ const whitelist = [ "video/webm", ]; +const uploadTokens = new Map(); + class Uploader { constructor(socket) { - const uploader = new SocketIOFileUploadServer(); - const folder = path.join(Helper.getFileUploadPath(), ".tmp"); + socket.on("upload:auth", () => { + const token = uuidv4(); - fsextra.ensureDir(folder, (err) => { - if (err) { - log.err(`Error ensuring ${folder} exists for uploads.`); - } else { - uploader.dir = folder; - } + uploadTokens.set(token, true); + + socket.emit("upload:auth", token); + + // Invalidate the token in one minute + setTimeout(() => uploadTokens.delete(token), 60 * 1000); }); - - uploader.on("complete", (data) => { - let randomName; - let destPath; - - // Generate a random file name for storage on disk - do { - randomName = crypto.randomBytes(8).toString("hex"); - destPath = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2), randomName); - } while (fs.existsSync(destPath)); - - fsextra.move(data.file.pathName, destPath).then(() => { - const slug = encodeURIComponent(path.basename(data.file.pathName)); - const url = `uploads/${randomName}/${slug}`; - socket.emit("upload:success", url); - }).catch(() => { - log.warn(`Unable to move uploaded file "${data.file.pathName}"`); - - // Emit failed upload to the client if file move fails - socket.emit("siofu_error", { - id: data.file.id, - message: "Unable to move uploaded file", - }); - }); - }); - - uploader.on("error", (data) => { - log.error(`File upload failed: ${data.error}`); - }); - - // maxFileSize is in bytes, but config option is passed in as KB - uploader.maxFileSize = Uploader.getMaxFileSize(); - uploader.listen(socket); } static isValidType(mimeType) { @@ -80,43 +50,173 @@ class Uploader { } static router(express) { - express.get("/uploads/:name/:slug*?", (req, res) => { - const name = req.params.name; + express.get("/uploads/:name/:slug*?", Uploader.routeGetFile); + express.post("/uploads/new/:token", Uploader.routeUploadFile); + } - const nameRegex = /^[0-9a-f]{16}$/; + static routeGetFile(req, res) { + const name = req.params.name; - if (!nameRegex.test(name)) { - return res.status(404).send("Not found"); + const nameRegex = /^[0-9a-f]{16}$/; + + if (!nameRegex.test(name)) { + return res.status(404).send("Not found"); + } + + const folder = name.substring(0, 2); + const uploadPath = Helper.getFileUploadPath(); + const filePath = path.join(uploadPath, folder, name); + const detectedMimeType = Uploader.getFileType(filePath); + + // doesn't exist + if (detectedMimeType === null) { + return res.status(404).send("Not found"); + } + + // Force a download in the browser if it's not a whitelisted type (binary or otherwise unknown) + const contentDisposition = Uploader.isValidType(detectedMimeType) ? "inline" : "attachment"; + + res.setHeader("Content-Disposition", contentDisposition); + res.setHeader("Cache-Control", "max-age=86400"); + res.contentType(detectedMimeType); + + return res.sendFile(filePath); + } + + static routeUploadFile(req, res) { + let busboyInstance; + let uploadUrl; + let randomName; + let destDir; + let destPath; + let streamWriter; + + const doneCallback = () => { + // detach the stream and drain any remaining data + if (busboyInstance) { + req.unpipe(busboyInstance); + req.on("readable", req.read.bind(req)); + + busboyInstance.removeAllListeners(); + busboyInstance = null; } - const folder = name.substring(0, 2); - const uploadPath = Helper.getFileUploadPath(); - const filePath = path.join(uploadPath, folder, name); - const detectedMimeType = Uploader.getFileType(filePath); + // close the output file stream + if (streamWriter) { + streamWriter.end(); + streamWriter = null; + } + }; - // doesn't exist - if (detectedMimeType === null) { - return res.status(404).send("Not found"); + const abortWithError = (err) => { + doneCallback(); + + // if we ended up erroring out, delete the output file from disk + if (destPath && fs.existsSync(destPath)) { + fs.unlinkSync(destPath); + destPath = null; } - // Force a download in the browser if it's not a whitelisted type (binary or otherwise unknown) - const contentDisposition = Uploader.isValidType(detectedMimeType) ? "inline" : "attachment"; + return res.status(400).json({error: err.message}); + }; - res.setHeader("Content-Disposition", contentDisposition); - res.setHeader("Cache-Control", "max-age=86400"); - res.contentType(detectedMimeType); + // if the authentication token is incorrect, bail out + if (uploadTokens.delete(req.params.token) !== true) { + return abortWithError(Error("Unauthorized")); + } - return res.sendFile(filePath); + // if the request does not contain any body data, bail out + if (req.headers["content-length"] < 1) { + return abortWithError(Error("Length Required")); + } + + // Only allow multipart, as busboy can throw an error on unsupported types + if (!req.headers["content-type"].startsWith("multipart/form-data")) { + return abortWithError(Error("Unsupported Content Type")); + } + + // create a new busboy processor, it is wrapped in try/catch + // because it can throw on malformed headers + try { + busboyInstance = new busboy({ + headers: req.headers, + limits: { + files: 1, // only allow one file per upload + fileSize: Uploader.getMaxFileSize(), + }, + }); + } catch (err) { + return abortWithError(err); + } + + // Any error or limit from busboy will abort the upload with an error + busboyInstance.on("error", abortWithError); + busboyInstance.on("partsLimit", () => abortWithError(Error("Parts limit reached"))); + busboyInstance.on("filesLimit", () => abortWithError(Error("Files limit reached"))); + busboyInstance.on("fieldsLimit", () => abortWithError(Error("Fields limit reached"))); + + // generate a random output filename for the file + // we use do/while loop to prevent the rare case of generating a file name + // that already exists on disk + do { + randomName = crypto.randomBytes(8).toString("hex"); + destDir = path.join(Helper.getFileUploadPath(), randomName.substring(0, 2)); + destPath = path.join(destDir, randomName); + } while (fs.existsSync(destPath)); + + // we split the filename into subdirectories (by taking 2 letters from the beginning) + // this helps avoid file system and certain tooling limitations when there are + // too many files on one folder + try { + fsextra.ensureDirSync(destDir); + } catch (err) { + log.err(`Error ensuring ${destDir} exists for uploads: ${err.message}`); + return abortWithError(err); + } + + // Open a file stream for writing + streamWriter = fs.createWriteStream(destPath); + streamWriter.on("error", abortWithError); + + busboyInstance.on("file", (fieldname, fileStream, filename) => { + uploadUrl = `uploads/${randomName}/${encodeURIComponent(filename)}`; + + // if the busboy data stream errors out or goes over the file size limit + // abort the processing with an error + fileStream.on("error", abortWithError); + fileStream.on("limit", () => { + fileStream.unpipe(streamWriter); + fileStream.on("readable", fileStream.read.bind(fileStream)); + + abortWithError(Error("File size limit reached")); + }); + + // Attempt to write the stream to file + fileStream.pipe(streamWriter); }); + + busboyInstance.on("finish", () => { + doneCallback(); + + // upload was done, send the generated file url to the client + res.status(200).json({ + url: uploadUrl, + }); + }); + + // pipe request body to busboy for processing + return req.pipe(busboyInstance); } static getMaxFileSize() { const configOption = Helper.config.fileUpload.maxFileSize; - if (configOption === -1) { // no file size limit - return null; + // Busboy uses Infinity to allow unlimited file size + if (configOption < 1) { + return Infinity; } + // maxFileSize is in bytes, but config option is passed in as KB return configOption * 1024; } @@ -129,14 +229,17 @@ class Uploader { // returns {ext, mime} if found, null if not. const file = fileType(buffer); + // if a file type was detected correctly, return it if (file) { return file.mime; } + // if the buffer is a valid UTF-8 buffer, use text/plain if (isUtf8(buffer)) { return "text/plain"; } + // otherwise assume it's random binary data return "application/octet-stream"; } catch (e) { if (e.code !== "ENOENT") { diff --git a/yarn.lock b/yarn.lock index 6c3b452e..7bbda454 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1641,6 +1641,13 @@ builtin-status-codes@^3.0.0: resolved "https://registry.yarnpkg.com/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz#85982878e21b98e1c66425e03d0174788f569ee8" integrity sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug= +busboy@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/busboy/-/busboy-0.3.0.tgz#6ee3cb1c844fc1f691d8f9d824f70128b3b5e485" + integrity sha512-e+kzZRAbbvJPLjQz2z+zAyr78BSi9IFeBTyLwF76g78Q2zRt/RZ1NtS3MS17v2yLqYfLz69zHdC+1L4ja8PwqQ== + dependencies: + dicer "0.3.0" + bytes@3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.0.0.tgz#d32815404d689699f85a4ea4fa8755dd13a96048" @@ -2693,6 +2700,13 @@ detect-node@^2.0.4: resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.4.tgz#014ee8f8f669c5c58023da64b8179c083a28c46c" integrity sha512-ZIzRpLJrOj7jjP2miAtgqIfmzbxa4ZOr5jJc601zklsfEx9oTzmmj2nVpIPRpNlRTIh8lc1kyViIY7BWSGNmKw== +dicer@0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/dicer/-/dicer-0.3.0.tgz#eacd98b3bfbf92e8ab5c2fdb71aaac44bb06b872" + integrity sha512-MdceRRWqltEG2dZqO769g27N/3PXfcKl04VhYnBlo2YhH7zPi88VebsjTKclaOyiuMaGU72hTfw3VkUitGcVCA== + dependencies: + streamsearch "0.1.2" + diff@3.5.0, diff@^3.5.0: version "3.5.0" resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" @@ -7710,11 +7724,6 @@ socket.io@2.2.0: socket.io-client "2.2.0" socket.io-parser "~3.3.0" -socketio-file-upload@0.7.0: - version "0.7.0" - resolved "https://registry.yarnpkg.com/socketio-file-upload/-/socketio-file-upload-0.7.0.tgz#aec62bb20051cbc6019792ff4dcc3907101eb466" - integrity sha512-eKL6LAPiDH9xIsn2eP0ZDzw/EUsnDiZxuDN6TpyPcsaS42Q5wt5VmeN6CGcPsMx57UMQ6jxjIljoBTnCVsG8Qw== - sockjs-client@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/sockjs-client/-/sockjs-client-1.3.0.tgz#12fc9d6cb663da5739d3dc5fb6e8687da95cb177" @@ -7962,6 +7971,11 @@ stream-shift@^1.0.0: resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952" integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI= +streamsearch@0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a" + integrity sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo= + strict-uri-encode@^1.0.0: version "1.1.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"