diff --git a/src/models/user.js b/src/models/user.js
index 68d251d1..e5898253 100644
--- a/src/models/user.js
+++ b/src/models/user.js
@@ -2,13 +2,16 @@ var _ = require("lodash");
 
 module.exports = User;
 
-function User(attr) {
-	// TODO: Remove this
-	attr.name = attr.name || attr.nick;
-	attr.mode = attr.mode || (attr.modes && attr.modes[0]) || "";
-
+function User(attr, prefixLookup) {
 	_.merge(this, _.extend({
-		mode: "",
-		name: ""
+		modes: [],
+		nick: ""
 	}, attr));
+
+	// irc-framework sets character mode, but lounge works with symbols
+	this.modes = this.modes.map(mode => prefixLookup[mode]);
+
+	// TODO: Remove this
+	this.name = this.nick;
+	this.mode = (this.modes && this.modes[0]) || "";
 }
diff --git a/src/plugins/irc-events/join.js b/src/plugins/irc-events/join.js
index d2984db2..69198904 100644
--- a/src/plugins/irc-events/join.js
+++ b/src/plugins/irc-events/join.js
@@ -17,7 +17,7 @@ module.exports = function(irc, network) {
 				chan: chan
 			});
 		}
-		chan.users.push(new User({nick: data.nick, modes: ""}));
+		chan.users.push(new User({nick: data.nick}));
 		chan.sortUsers(irc);
 		client.emit("users", {
 			chan: chan.id
diff --git a/src/plugins/irc-events/mode.js b/src/plugins/irc-events/mode.js
index a466be2d..886c42fb 100644
--- a/src/plugins/irc-events/mode.js
+++ b/src/plugins/irc-events/mode.js
@@ -17,17 +17,19 @@ module.exports = function(irc, network) {
 		}
 
 		var usersUpdated;
+		var supportsMultiPrefix = network.irc.network.cap.isEnabled("multi-prefix");
+		var userModeSortPriority = {};
+
+		irc.network.options.PREFIX.forEach(function(prefix, index) {
+			userModeSortPriority[prefix.symbol] = index;
+		});
 
 		for (var i = 0; i < data.modes.length; i++) {
 			var mode = data.modes[i];
 			var text = mode.mode;
+
 			if (mode.param) {
 				text += " " + mode.param;
-
-				var user = _.find(targetChan.users, {name: mode.param});
-				if (typeof user !== "undefined") {
-					usersUpdated = true;
-				}
 			}
 
 			var msg = new Msg({
@@ -39,11 +41,51 @@ module.exports = function(irc, network) {
 				self: data.nick === irc.user.nick
 			});
 			targetChan.pushMessage(client, msg);
+
+			if (!mode.param) {
+				continue;
+			}
+
+			var user = _.find(targetChan.users, {name: mode.param});
+			if (!user) {
+				continue;
+			}
+
+			usersUpdated = true;
+
+			if (!supportsMultiPrefix) {
+				continue;
+			}
+
+			var add = mode.mode[0] === "+";
+			var changedMode = network.prefixLookup[mode.mode[1]];
+
+			if (!add) {
+				_.pull(user.modes, changedMode);
+			} else if (user.modes.indexOf(changedMode) === -1) {
+				user.modes.push(changedMode);
+				user.modes.sort(function(a, b) {
+					return userModeSortPriority[a] - userModeSortPriority[b];
+				});
+			}
+
+			// TODO: remove in future
+			user.mode = (user.modes && user.modes[0]) || "";
 		}
 
-		if (usersUpdated) {
+		if (!usersUpdated) {
+			return;
+		}
+
+		if (!supportsMultiPrefix) {
 			// TODO: This is horrible
 			irc.raw("NAMES", data.target);
+		} else {
+			targetChan.sortUsers(irc);
+
+			client.emit("users", {
+				chan: targetChan.id
+			});
 		}
 	});
 };
diff --git a/src/plugins/irc-events/names.js b/src/plugins/irc-events/names.js
index 4307ff22..6bee2696 100644
--- a/src/plugins/irc-events/names.js
+++ b/src/plugins/irc-events/names.js
@@ -8,18 +8,17 @@ module.exports = function(irc, network) {
 		if (typeof chan === "undefined") {
 			return;
 		}
-		chan.users = [];
-		_.each(data.users, function(u) {
-			var user = new User(u);
 
-			// irc-framework sets characater mode, but lounge works with symbols
-			if (user.mode) {
-				user.mode = network.prefixLookup[user.mode];
-			}
+		chan.users = [];
+
+		_.each(data.users, function(u) {
+			var user = new User(u, network.prefixLookup);
 
 			chan.users.push(user);
 		});
+
 		chan.sortUsers(irc);
+
 		client.emit("users", {
 			chan: chan.id
 		});
diff --git a/test/models/chan.js b/test/models/chan.js
index fe85f927..a12335c0 100644
--- a/test/models/chan.js
+++ b/test/models/chan.js
@@ -1,36 +1,48 @@
 "use strict";
 
+var _ = require("lodash");
 var expect = require("chai").expect;
 
 var Chan = require("../../src/models/chan");
 var User = require("../../src/models/user");
 
-function makeUser(name) {
-	// TODO Update/Fix this when User constructor gets reworked (see its TODO)
-	return new User({nick: name, mode: ""});
-}
-
-function getUserNames(chan) {
-	return chan.users.map(function(u) {
-		return u.name;
-	});
-}
-
 describe("Chan", function() {
 	describe("#sortUsers(irc)", function() {
-		var fullNetworkPrefix = [
-			{symbol: "~", mode: "q"},
-			{symbol: "&", mode: "a"},
-			{symbol: "@", mode: "o"},
-			{symbol: "%", mode: "h"},
-			{symbol: "+", mode: "v"}
-		];
+		var network = {
+			network: {
+				options: {
+					PREFIX: [
+						{symbol: "~", mode: "q"},
+						{symbol: "&", mode: "a"},
+						{symbol: "@", mode: "o"},
+						{symbol: "%", mode: "h"},
+						{symbol: "+", mode: "v"}
+					]
+				}
+			}
+		};
+
+		var prefixLookup = {};
+
+		_.each(network.network.options.PREFIX, function(mode) {
+			prefixLookup[mode.mode] = mode.symbol;
+		});
+
+		var makeUser = function(nick) {
+			return new User({nick: nick}, prefixLookup);
+		};
+
+		var getUserNames = function(chan) {
+			return chan.users.map(function(u) {
+				return u.name;
+			});
+		};
 
 		it("should sort a simple user list", function() {
 			var chan = new Chan({users: [
 				"JocelynD", "YaManicKill", "astorije", "xPaw", "Max-P"
 			].map(makeUser)});
-			chan.sortUsers({network: {options: {PREFIX: fullNetworkPrefix}}});
+			chan.sortUsers(network);
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"astorije", "JocelynD", "Max-P", "xPaw", "YaManicKill"
@@ -39,13 +51,13 @@ describe("Chan", function() {
 
 		it("should group users by modes", function() {
 			var chan = new Chan({users: [
-				new User({name: "JocelynD", mode: "&"}),
-				new User({name: "YaManicKill", mode: "+"}),
-				new User({name: "astorije", mode: "%"}),
-				new User({name: "xPaw", mode: "~"}),
-				new User({name: "Max-P", mode: "@"}),
+				new User({nick: "JocelynD", modes: ["a", "o"]}, prefixLookup),
+				new User({nick: "YaManicKill", modes: ["v"]}, prefixLookup),
+				new User({nick: "astorije", modes: ["h"]}, prefixLookup),
+				new User({nick: "xPaw", modes: ["q"]}, prefixLookup),
+				new User({nick: "Max-P", modes: ["o"]}, prefixLookup),
 			]});
-			chan.sortUsers({network: {options: {PREFIX: fullNetworkPrefix}}});
+			chan.sortUsers(network);
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"xPaw", "JocelynD", "Max-P", "astorije", "YaManicKill"
@@ -54,13 +66,13 @@ describe("Chan", function() {
 
 		it("should sort a mix of users and modes", function() {
 			var chan = new Chan({users: [
-				new User({name: "JocelynD"}),
-				new User({name: "YaManicKill", mode: "@"}),
-				new User({name: "astorije"}),
-				new User({name: "xPaw"}),
-				new User({name: "Max-P", mode: "@"}),
+				new User({nick: "JocelynD"}, prefixLookup),
+				new User({nick: "YaManicKill", modes: ["o"]}, prefixLookup),
+				new User({nick: "astorije"}, prefixLookup),
+				new User({nick: "xPaw"}, prefixLookup),
+				new User({nick: "Max-P", modes: ["o"]}, prefixLookup),
 			]});
-			chan.sortUsers({network: {options: {PREFIX: fullNetworkPrefix}}});
+			chan.sortUsers(network);
 
 			expect(getUserNames(chan)).to.deep.equal(
 				["Max-P", "YaManicKill", "astorije", "JocelynD", "xPaw"]
@@ -69,7 +81,7 @@ describe("Chan", function() {
 
 		it("should be case-insensitive", function() {
 			var chan = new Chan({users: ["aB", "Ad", "AA", "ac"].map(makeUser)});
-			chan.sortUsers({network: {options: {PREFIX: fullNetworkPrefix}}});
+			chan.sortUsers(network);
 
 			expect(getUserNames(chan)).to.deep.equal(["AA", "aB", "ac", "Ad"]);
 		});
@@ -79,7 +91,7 @@ describe("Chan", function() {
 				"[foo", "]foo", "(foo)", "{foo}", "<foo>", "_foo", "@foo", "^foo",
 				"&foo", "!foo", "+foo", "Foo"
 			].map(makeUser)});
-			chan.sortUsers({network: {options: {PREFIX: fullNetworkPrefix}}});
+			chan.sortUsers(network);
 
 			expect(getUserNames(chan)).to.deep.equal([
 				"!foo", "&foo", "(foo)", "+foo", "<foo>", "@foo", "[foo", "]foo",