diff --git a/src/client.js b/src/client.js
index 9e9c707c..a8bc3e8b 100644
--- a/src/client.js
+++ b/src/client.js
@@ -134,7 +134,7 @@ Client.prototype.find = function(id) {
 };
 
 Client.prototype.connect = function(args) {
-	var config = Helper.getConfig();
+	var config = Helper.config;
 	var client = this;
 
 	var nick = args.nick || "lounge-user";
@@ -426,9 +426,8 @@ Client.prototype.quit = function() {
 var timer;
 Client.prototype.save = function(force) {
 	var client = this;
-	var config = Helper.getConfig();
 
-	if (config.public) {
+	if (Helper.config.public) {
 		return;
 	}
 
diff --git a/src/clientManager.js b/src/clientManager.js
index 10cd60f5..a9b61bfd 100644
--- a/src/clientManager.js
+++ b/src/clientManager.js
@@ -7,12 +7,10 @@ var Oidentd = require("./oidentd");
 module.exports = ClientManager;
 
 function ClientManager() {
-	var config = Helper.getConfig();
-
 	this.clients = [];
 
-	if (typeof config.oidentd === "string") {
-		this.identHandler = new Oidentd(config.oidentd);
+	if (typeof Helper.config.oidentd === "string") {
+		this.identHandler = new Oidentd(Helper.config.oidentd);
 	}
 }
 
diff --git a/src/command-line/start.js b/src/command-line/start.js
index 58f7a4ba..e6c37ef1 100644
--- a/src/command-line/start.js
+++ b/src/command-line/start.js
@@ -13,22 +13,25 @@ program
 	.description("Start the server")
 	.action(function() {
 		var users = new ClientManager().getUsers();
-		var config = Helper.getConfig();
-		var mode = config.public;
+
+		var mode = Helper.config.public;
 		if (program.public) {
 			mode = true;
 		} else if (program.private) {
 			mode = false;
 		}
+
 		if (!mode && !users.length) {
 			log.warn("No users found!");
 			log.info("Create a new user with 'lounge add <name>'.");
-		} else {
-			server({
-				host: program.host || process.env.IP   || config.host,
-				port: program.port || process.env.PORT || config.port,
-				bind: program.bind || config.bind,
-				public: mode
-			});
+
+			return;
 		}
+
+		Helper.config.host = program.host || Helper.config.host;
+		Helper.config.port = program.port || Helper.config.port;
+		Helper.config.bind = program.bind || Helper.config.bind;
+		Helper.config.public = mode;
+
+		server();
 	});
diff --git a/src/helper.js b/src/helper.js
index b6d11cad..cac3ce78 100644
--- a/src/helper.js
+++ b/src/helper.js
@@ -1,9 +1,10 @@
+var _ = require("lodash");
 var path = require("path");
 var os = require("os");
 
 var Helper = {
+	config: null,
 	expandHome: expandHome,
-	getConfig: getConfig,
 	getUserConfigPath: getUserConfigPath,
 	getUserLogsPath: getUserLogsPath,
 	setHome: setHome,
@@ -11,10 +12,21 @@ var Helper = {
 
 module.exports = Helper;
 
+Helper.config = require(path.resolve(path.join(
+	__dirname,
+	"..",
+	"defaults",
+	"config.js"
+)));
+
 function setHome(homePath) {
 	this.HOME = expandHome(homePath || "~/.lounge");
 	this.CONFIG_PATH = path.join(this.HOME, "config.js");
 	this.USERS_PATH = path.join(this.HOME, "users");
+
+	// Reload config from new home location
+	var userConfig = require(this.CONFIG_PATH);
+	this.config = _.extend(this.config, userConfig);
 }
 
 function getUserConfigPath(name) {
@@ -25,10 +37,6 @@ function getUserLogsPath(name, network) {
 	return path.join(this.HOME, "logs", name, network);
 }
 
-function getConfig() {
-	return require(this.CONFIG_PATH);
-}
-
 function expandHome(shortenedPath) {
 	var home;
 
diff --git a/src/log.js b/src/log.js
index eb792bfe..153f3a3d 100644
--- a/src/log.js
+++ b/src/log.js
@@ -3,9 +3,8 @@ var moment = require("moment");
 var Helper = require("./helper");
 
 function timestamp(type, messageArgs) {
-	var config = Helper.getConfig();
-	var format = (config.logs || {}).format || "YYYY-MM-DD HH:mm:ss";
-	var tz = (config.logs || {}).timezone || "UTC+00:00";
+	var format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss";
+	var tz = Helper.config.logs.timezone || "UTC+00:00";
 
 	var time = moment().utcOffset(tz).format(format);
 
diff --git a/src/models/chan.js b/src/models/chan.js
index 6bcf51b2..f5481141 100644
--- a/src/models/chan.js
+++ b/src/models/chan.js
@@ -10,7 +10,6 @@ Chan.Type = {
 };
 
 var id = 0;
-var config = Helper.getConfig();
 
 function Chan(attr) {
 	_.merge(this, _.extend({
@@ -33,14 +32,14 @@ Chan.prototype.pushMessage = function(client, msg) {
 
 	// Never store messages in public mode as the session
 	// is completely destroyed when the page gets closed
-	if (config.public) {
+	if (Helper.config.public) {
 		return;
 	}
 
 	this.messages.push(msg);
 
-	if (config.maxHistory >= 0 && this.messages.length > config.maxHistory) {
-		this.messages.splice(0, this.messages.length - config.maxHistory);
+	if (Helper.config.maxHistory >= 0 && this.messages.length > Helper.config.maxHistory) {
+		this.messages.splice(0, this.messages.length - Helper.config.maxHistory);
 	}
 };
 
diff --git a/src/plugins/irc-events/link.js b/src/plugins/irc-events/link.js
index 15ad92d4..7fdd9c05 100644
--- a/src/plugins/irc-events/link.js
+++ b/src/plugins/irc-events/link.js
@@ -10,8 +10,7 @@ process.setMaxListeners(0);
 module.exports = function(irc, network) {
 	var client = this;
 	irc.on("privmsg", function(data) {
-		var config = Helper.getConfig();
-		if (!config.prefetch) {
+		if (!Helper.config.prefetch) {
 			return;
 		}
 
@@ -45,7 +44,6 @@ module.exports = function(irc, network) {
 };
 
 function parse(msg, url, res, client) {
-	var config = Helper.getConfig();
 	var toggle = msg.toggle = {
 		id: msg.id,
 		type: "",
@@ -55,9 +53,6 @@ function parse(msg, url, res, client) {
 		link: url
 	};
 
-	if (!config.prefetchMaxImageSize) {
-		config.prefetchMaxImageSize = 512;
-	}
 	switch (res.type) {
 	case "text/html":
 		var $ = cheerio.load(res.text);
@@ -77,7 +72,7 @@ function parse(msg, url, res, client) {
 	case "image/gif":
 	case "image/jpg":
 	case "image/jpeg":
-		if (res.size < (config.prefetchMaxImageSize * 1024)) {
+		if (res.size < (Helper.config.prefetchMaxImageSize * 1024)) {
 			toggle.type = "image";
 		} else {
 			return;
diff --git a/src/server.js b/src/server.js
index 01afe3dc..03f7436e 100644
--- a/src/server.js
+++ b/src/server.js
@@ -8,39 +8,32 @@ var fs = require("fs");
 var io = require("socket.io");
 var dns = require("dns");
 var Helper = require("./helper");
-var config = {};
 
 var manager = null;
 
-module.exports = function(options) {
+module.exports = function() {
 	manager = new ClientManager();
-	config = Helper.getConfig();
-	config = _.extend(config, options);
 
 	var app = express()
 		.use(allRequests)
 		.use(index)
 		.use(express.static("client"));
 
+	var config = Helper.config;
 	var server = null;
-	var https = config.https || {};
-	var protocol = https.enable ? "https" : "http";
-	var port = config.port;
-	var host = config.host;
-	var transports = config.transports || ["polling", "websocket"];
 
-	if (!https.enable) {
+	if (!config.https.enable) {
 		server = require("http");
-		server = server.createServer(app).listen(port, host);
+		server = server.createServer(app).listen(config.port, config.host);
 	} else {
 		server = require("spdy");
 		server = server.createServer({
-			key: fs.readFileSync(Helper.expandHome(https.key)),
-			cert: fs.readFileSync(Helper.expandHome(https.certificate))
-		}, app).listen(port, host);
+			key: fs.readFileSync(Helper.expandHome(config.https.key)),
+			cert: fs.readFileSync(Helper.expandHome(config.https.certificate))
+		}, app).listen(config.port, config.host);
 	}
 
-	if ((config.identd || {}).enable) {
+	if (config.identd.enable) {
 		if (manager.identHandler) {
 			log.warn("Using both identd and oidentd at the same time!");
 		}
@@ -49,7 +42,7 @@ module.exports = function(options) {
 	}
 
 	var sockets = io(server, {
-		transports: transports
+		transports: config.transports
 	});
 
 	sockets.on("connect", function(socket) {
@@ -62,7 +55,8 @@ module.exports = function(options) {
 
 	manager.sockets = sockets;
 
-	log.info("The Lounge v" + pkg.version + " is now running on", protocol + "://" + (config.host || "*") + ":" + config.port + "/");
+	var protocol = config.https.enable ? "https" : "http";
+	log.info("The Lounge v" + pkg.version + " is now running on", protocol + "://" + (config.host || "*") + ":" + config.port + "/", (config.public ? "in public mode" : "in private mode"));
 	log.info("Press ctrl-c to stop\n");
 
 	if (!config.public) {
@@ -74,7 +68,7 @@ module.exports = function(options) {
 };
 
 function getClientIp(req) {
-	if (!config.reverseProxy) {
+	if (!Helper.config.reverseProxy) {
 		return req.connection.remoteAddress;
 	} else {
 		return req.headers["x-forwarded-for"] || req.connection.remoteAddress;
@@ -94,7 +88,7 @@ function index(req, res, next) {
 	return fs.readFile("client/index.html", "utf-8", function(err, file) {
 		var data = _.merge(
 			pkg,
-			config
+			Helper.config
 		);
 		var template = _.template(file);
 		res.setHeader("Content-Security-Policy", "default-src *; style-src * 'unsafe-inline'; script-src 'self'; child-src 'none'; object-src 'none'; form-action 'none'; referrer no-referrer;");
@@ -130,7 +124,7 @@ function init(socket, client) {
 				client.connect(data);
 			}
 		);
-		if (!config.public) {
+		if (!Helper.config.public) {
 			socket.on(
 				"change-password",
 				function(data) {
@@ -217,14 +211,14 @@ function reverseDnsLookup(socket, client) {
 
 function auth(data) {
 	var socket = this;
-	if (config.public) {
+	if (Helper.config.public) {
 		var client = new Client(manager);
 		manager.clients.push(client);
 		socket.on("disconnect", function() {
 			manager.clients = _.without(manager.clients, client);
 			client.quit();
 		});
-		if (config.webirc) {
+		if (Helper.config.webirc) {
 			reverseDnsLookup(socket, client);
 		} else {
 			init(socket, client);
@@ -242,7 +236,7 @@ function auth(data) {
 				}
 			}
 			if (success) {
-				if (config.webirc !== null && !client.config["ip"]) {
+				if (Helper.config.webirc !== null && !client.config["ip"]) {
 					reverseDnsLookup(socket, client);
 				} else {
 					init(socket, client);
diff --git a/src/userLog.js b/src/userLog.js
index 8bb3e6d5..2d4e4ba1 100644
--- a/src/userLog.js
+++ b/src/userLog.js
@@ -12,9 +12,8 @@ module.exports.write = function(user, network, chan, msg) {
 		return;
 	}
 
-	var config = Helper.getConfig();
-	var format = (config.logs || {}).format || "YYYY-MM-DD HH:mm:ss";
-	var tz = (config.logs || {}).timezone || "UTC+00:00";
+	var format = Helper.config.logs.format || "YYYY-MM-DD HH:mm:ss";
+	var tz = Helper.config.logs.timezone || "UTC+00:00";
 
 	var time = moment().utcOffset(tz).format(format);
 	var line = "[" + time + "] ";