diff --git a/config/config.go b/config/config.go
index b83cf27..ee82fea 100644
--- a/config/config.go
+++ b/config/config.go
@@ -4,23 +4,48 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"net"
 	"os"
 
 	"github.com/google/shlex"
 )
 
+type IPSet []*net.IPNet
+
+func (set IPSet) Contains(ip net.IP) bool {
+	for _, n := range set {
+		if n.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// loopbackIPs contains the loopback networks 127.0.0.0/8 and ::1/128.
+var loopbackIPs = IPSet{
+	&net.IPNet{
+		IP:   net.IP{127, 0, 0, 0},
+		Mask: net.CIDRMask(8, 32),
+	},
+	&net.IPNet{
+		IP:   net.IPv6loopback,
+		Mask: net.CIDRMask(128, 128),
+	},
+}
+
 type TLS struct {
 	CertPath, KeyPath string
 }
 
 type Server struct {
-	Listen      []string
-	Hostname    string
-	TLS         *TLS
-	SQLDriver   string
-	SQLSource   string
-	LogPath     string
-	HTTPOrigins []string
+	Listen         []string
+	Hostname       string
+	TLS            *TLS
+	SQLDriver      string
+	SQLSource      string
+	LogPath        string
+	HTTPOrigins    []string
+	AcceptProxyIPs IPSet
 }
 
 func Defaults() *Server {
@@ -29,9 +54,10 @@ func Defaults() *Server {
 		hostname = "localhost"
 	}
 	return &Server{
-		Hostname:  hostname,
-		SQLDriver: "sqlite3",
-		SQLSource: "soju.db",
+		Hostname:       hostname,
+		SQLDriver:      "sqlite3",
+		SQLSource:      "soju.db",
+		AcceptProxyIPs: loopbackIPs,
 	}
 }
 
@@ -93,6 +119,15 @@ func Parse(r io.Reader) (*Server, error) {
 			}
 		case "http-origin":
 			srv.HTTPOrigins = append(srv.HTTPOrigins, d.Params...)
+		case "accept-proxy-ip":
+			srv.AcceptProxyIPs = nil
+			for _, s := range d.Params {
+				_, n, err := net.ParseCIDR(s)
+				if err != nil {
+					return nil, fmt.Errorf("directive %q: failed to parse CIDR: %v", d.Name, err)
+				}
+				srv.AcceptProxyIPs = append(srv.AcceptProxyIPs, n)
+			}
 		default:
 			return nil, fmt.Errorf("unknown directive %q", d.Name)
 		}
diff --git a/doc/soju.1.scd b/doc/soju.1.scd
index 5cfbaef..c8aad7c 100644
--- a/doc/soju.1.scd
+++ b/doc/soju.1.scd
@@ -109,6 +109,12 @@ The following directives are supported:
 	List of allowed HTTP origins for WebSocket listeners. The parameters are
 	interpreted as shell patterns, see *glob*(7).
 
+*accept-proxy-ip* <cidr...>
+	Allow the specified IPs to act as a proxy. Proxys have the ability to
+	overwrite the remote and local connection addresses (via the X-Forwarded-*
+	HTTP header fields). By default, the loopback addresses 127.0.0.0/8 and
+	::1/128 are accepted.
+
 # IRC SERVICE
 
 soju exposes an IRC service called *BouncerServ* to manage the bouncer.
diff --git a/server.go b/server.go
index 67aa8d4..99cec0f 100644
--- a/server.go
+++ b/server.go
@@ -11,6 +11,8 @@ import (
 
 	"gopkg.in/irc.v3"
 	"nhooyr.io/websocket"
+
+	"git.sr.ht/~emersion/soju/config"
 )
 
 // TODO: make configurable
@@ -41,13 +43,14 @@ func (l *prefixLogger) Printf(format string, v ...interface{}) {
 }
 
 type Server struct {
-	Hostname     string
-	Logger       Logger
-	RingCap      int
-	HistoryLimit int
-	LogPath      string
-	Debug        bool
-	HTTPOrigins  []string
+	Hostname       string
+	Logger         Logger
+	RingCap        int
+	HistoryLimit   int
+	LogPath        string
+	Debug          bool
+	HTTPOrigins    []string
+	AcceptProxyIPs config.IPSet
 
 	db *DB
 
@@ -153,19 +156,19 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 		return
 	}
 
-	isLoopback := false
+	isProxy := false
 	if host, _, err := net.SplitHostPort(req.RemoteAddr); err == nil {
 		if ip := net.ParseIP(host); ip != nil {
-			isLoopback = ip.IsLoopback()
+			isProxy = s.AcceptProxyIPs.Contains(ip)
 		}
 	}
 
-	// Only trust X-Forwarded-* header fields if this is a loopback connection,
+	// Only trust X-Forwarded-* header fields if this is a trusted proxy IP
 	// to prevent users from spoofing the remote address
 	remoteAddr := req.RemoteAddr
 	forwardedHost := req.Header.Get("X-Forwarded-For")
 	forwardedPort := req.Header.Get("X-Forwarded-Port")
-	if isLoopback && forwardedHost != "" && forwardedPort != "" {
+	if isProxy && forwardedHost != "" && forwardedPort != "" {
 		remoteAddr = net.JoinHostPort(forwardedHost, forwardedPort)
 	}