Add WHO cache

This adds a new field to upstreams, members, which is a casemapped map
of upstream users known to the soju. The upstream users known to soju
are: self, any monitored user, and any user with whom we share a
channel.

The information stored for each upstream user corresponds to the info
that can be returned by a WHO/WHOX command.

We build the upstream user information both incrementally, capturing
information contained in JOIN and AWAY messages; and with the bulk user
information contained in WHO replies we receive.

This lets us build a user cache that can then be used to return
synthetic WHO responses to later WHO requests by downstreams.

This is useful because some networks (eg Libera) heavily throttle WHO
commands, and without this cache, any downstream connecting would send 1
WHO command per channel, so possibly more than a dozen WHO commands,
which soju then forwarded to the upstream as WHO commands.

With this cache most WHO commands can be cached and avoid sending
WHO commands to the upstream.

In order to cache the "flags" field, we synthetize the field from user
info we get from incremental messages: away status (H/G) and bot status
(B). This could result in incorrect values for proprietary user fields.
Support for the server-operator status (*) is also not supported.

Of note is that it is difficult to obtain a user "connected server"
field incrementally, so clients that want to maximize their WHO cache
hit ratio can use WHOX to only request fields they need, and in
particular not include the server field flag.

Co-authored-by: delthas <delthas@dille.cc>

downstream.go

@@ -2079,16 +2079,7 @@ func (dc *downstreamConn) handleMessageRegistered(ctx context.Context, msg *irc.
 			options = msg.Params[1]
 		}
 
-		optionsParts := strings.SplitN(options, "%", 2)
+		fields, whoxToken := xirc.ParseWHOXOptions(options)
-		// TODO: add support for WHOX flags in optionsParts[0]
-		var fields, whoxToken string
-		if len(optionsParts) == 2 {
-			optionsParts := strings.SplitN(optionsParts[1], ",", 2)
-			fields = strings.ToLower(optionsParts[0])
-			if len(optionsParts) == 2 && strings.Contains(fields, "t") {
-				whoxToken = optionsParts[1]
-			}
-		}
 
 		// TODO: support mixed bouncer/upstream WHO queries
 		maskCM := casemapASCII(mask)
@@ -2157,6 +2148,29 @@ func (dc *downstreamConn) handleMessageRegistered(ctx context.Context, msg *irc.
 			return err
 		}
 
+		// Check if we have the reply cached
+		if l, ok := uc.getCachedWHO(mask, fields); ok {
+			for _, uu := range l {
+				info := xirc.WHOXInfo{
+					Token:    whoxToken,
+					Username: uu.Username,
+					Hostname: uu.Hostname,
+					Server:   uu.Server,
+					Nickname: uu.Nickname,
+					Flags:    uu.Flags,
+					Account:  uu.Account,
+					Realname: uu.Realname,
+				}
+				dc.SendMessage(xirc.GenerateWHOXReply(dc.srv.prefix(), dc.nick, fields, &info))
+			}
+			dc.SendMessage(&irc.Message{
+				Prefix:  dc.srv.prefix(),
+				Command: irc.RPL_ENDOFWHO,
+				Params:  []string{dc.nick, endOfWhoToken, "End of /WHO list"},
+			})
+			return nil
+		}
+
 		uc.enqueueCommand(dc, msg)
 	case "WHOIS":
 		if len(msg.Params) == 0 {

irc.go

@@ -379,6 +379,20 @@ func (cm *upstreamChannelCasemapMap) ForEach(f func(*upstreamChannel)) {
 	}
 }
 
+type upstreamUserCasemapMap struct{ casemapMap }
+
+func (cm *upstreamUserCasemapMap) Get(name string) *upstreamUser {
+	if v := cm.get(name); v == nil {
+		return nil
+	} else {
+		return v.(*upstreamUser)
+	}
+}
+
+func (cm *upstreamUserCasemapMap) Set(u *upstreamUser) {
+	cm.set(u.Nickname, u)
+}
+
 type channelCasemapMap struct{ casemapMap }
 
 func (cm *channelCasemapMap) Get(name string) *database.Channel {

upstream.go

@@ -111,6 +111,69 @@ type upstreamBatch struct {
 	Label  string
 }
 
+type upstreamUser struct {
+	Nickname string
+	Username string
+	Hostname string
+	Server   string
+	Flags    string
+	Account  string
+	Realname string
+}
+
+func (uu *upstreamUser) hasWHOXFields(fields string) bool {
+	for i := 0; i < len(fields); i++ {
+		ok := false
+		switch fields[i] {
+		case 'n':
+			ok = uu.Nickname != ""
+		case 'u':
+			ok = uu.Username != ""
+		case 'h':
+			ok = uu.Hostname != ""
+		case 's':
+			ok = uu.Server != ""
+		case 'f':
+			ok = uu.Flags != ""
+		case 'a':
+			ok = uu.Account != ""
+		case 'r':
+			ok = uu.Realname != ""
+		case 't', 'c', 'i', 'd', 'l', 'o':
+			// we return static values for those fields, so they are always available
+			ok = true
+		}
+		if !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (uu *upstreamUser) updateFrom(update *upstreamUser) {
+	if update.Nickname != "" {
+		uu.Nickname = update.Nickname
+	}
+	if update.Username != "" {
+		uu.Username = update.Username
+	}
+	if update.Hostname != "" {
+		uu.Hostname = update.Hostname
+	}
+	if update.Server != "" {
+		uu.Server = update.Server
+	}
+	if update.Flags != "" {
+		uu.Flags = update.Flags
+	}
+	if update.Account != "" {
+		uu.Account = update.Account
+	}
+	if update.Realname != "" {
+		uu.Realname = update.Realname
+	}
+}
+
 type pendingUpstreamCommand struct {
 	downstreamID uint64
 	msg          *irc.Message
@@ -138,6 +201,7 @@ type upstreamConn struct {
 	hostname    string
 	modes       userModes
 	channels    upstreamChannelCasemapMap
+	users       upstreamUserCasemapMap
 	caps        xirc.CapRegistry
 	batches     map[string]upstreamBatch
 	away        bool
@@ -263,6 +327,7 @@ func connectToUpstream(ctx context.Context, network *network) (*upstreamConn, er
 		network:               network,
 		user:                  network.user,
 		channels:              upstreamChannelCasemapMap{newCasemapMap()},
+		users:                 upstreamUserCasemapMap{newCasemapMap()},
 		caps:                  xirc.NewCapRegistry(),
 		batches:               make(map[string]upstreamBatch),
 		serverPrefix:          &irc.Prefix{Name: "*"},
@@ -973,6 +1038,10 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 			}
 		})
 
+		uc.cacheUserInfo(msg.Prefix.Name, &upstreamUser{
+			Nickname: newNick,
+		})
+
 		if !me {
 			uc.forEachDownstream(func(dc *downstreamConn) {
 				dc.SendMessage(msg)
@@ -989,6 +1058,10 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 			return err
 		}
 
+		uc.cacheUserInfo(msg.Prefix.Name, &upstreamUser{
+			Realname: newRealname,
+		})
+
 		// TODO: consider appending this message to logs
 
 		if uc.isOurNick(msg.Prefix.Name) {
@@ -1035,6 +1108,30 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 			return err
 		}
 
+		uu := &upstreamUser{
+			Username: msg.Prefix.User,
+			Hostname: msg.Prefix.Host,
+		}
+		if uc.caps.IsEnabled("away-notify") {
+			// we have enough info to build the user flags in a best-effort manner:
+			// - the H/G flag is set to Here first, will be replaced by Gone later if the user is AWAY
+			uu.Flags = "H"
+			// - the B (bot mode) flag is set if the JOIN comes from a bot
+			//   note: we have no way to track the user bot mode after they have joined
+			//         (we are not notified of the bot mode updates), but this is good enough.
+			if _, ok := msg.Tags["bot"]; ok {
+				if bot := uc.isupport["BOT"]; bot != nil {
+					uu.Flags += *bot
+				}
+			}
+			// TODO: add the server operator flag (`*`) if the message has an oper-tag
+		}
+		if len(msg.Params) > 2 { // extended-join
+			uu.Account = msg.Params[1]
+			uu.Realname = msg.Params[2]
+		}
+		uc.cacheUserInfo(msg.Prefix.Name, uu)
+
 		for _, ch := range strings.Split(channels, ",") {
 			if uc.isOurNick(msg.Prefix.Name) {
 				uc.logger.Printf("joined channel %q", ch)
@@ -1075,6 +1172,11 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 				if uch := uc.channels.Get(ch); uch != nil {
 					uc.channels.Del(ch)
 					uch.updateAutoDetach(0)
+					uch.Members.ForEach(func(nick string, memberships *xirc.MembershipSet) {
+						if !uc.shouldCacheUserInfo(nick) {
+							uc.users.Del(nick)
+						}
+					})
 				}
 			} else {
 				ch, err := uc.getChannel(ch)
@@ -1082,6 +1184,9 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 					return err
 				}
 				ch.Members.Del(msg.Prefix.Name)
+				if !uc.shouldCacheUserInfo(msg.Prefix.Name) {
+					uc.users.Del(msg.Prefix.Name)
+				}
 			}
 
 			chMsg := msg.Copy()
@@ -1096,13 +1201,23 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 
 		if uc.isOurNick(user) {
 			uc.logger.Printf("kicked from channel %q by %s", channel, msg.Prefix.Name)
+			if uch := uc.channels.Get(channel); uch != nil {
 				uc.channels.Del(channel)
+				uch.Members.ForEach(func(nick string, memberships *xirc.MembershipSet) {
+					if !uc.shouldCacheUserInfo(nick) {
+						uc.users.Del(nick)
+					}
+				})
+			}
 		} else {
 			ch, err := uc.getChannel(channel)
 			if err != nil {
 				return err
 			}
 			ch.Members.Del(user)
+			if !uc.shouldCacheUserInfo(user) {
+				uc.users.Del(user)
+			}
 		}
 
 		uc.produce(channel, msg, 0)
@@ -1118,6 +1233,8 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 			}
 		})
 
+		uc.users.Del(msg.Prefix.Name)
+
 		if msg.Prefix.Name != uc.nick {
 			uc.forEachDownstream(func(dc *downstreamConn) {
 				dc.SendMessage(msg)
@@ -1358,15 +1475,68 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 				forwardChannel(ctx, dc, ch)
 			})
 		}
-	case irc.RPL_WHOREPLY, xirc.RPL_WHOSPCRPL:
+	case irc.RPL_WHOREPLY:
+		var username, host, server, nick, flags, trailing string
+		if err := parseMessageParams(msg, nil, nil, &username, &host, &server, &nick, &flags, &trailing); err != nil {
+			return err
+		}
+
 		dc, cmd := uc.currentPendingCommand("WHO")
 		if cmd == nil {
-			return fmt.Errorf("unexpected WHO reply %v: no matching pending WHO", msg.Command)
+			return fmt.Errorf("unexpected RPL_WHOREPLY: no matching pending WHO")
+		} else if dc == nil {
+			return nil
+		}
+
+		parts := strings.SplitN(trailing, " ", 2)
+		if len(parts) != 2 {
+			return fmt.Errorf("malformed RPL_WHOREPLY: failed to parse real name")
+		}
+		realname := parts[1]
+
+		dc.SendMessage(msg)
+
+		if uc.shouldCacheUserInfo(nick) {
+			uc.cacheUserInfo(nick, &upstreamUser{
+				Username: username,
+				Hostname: host,
+				Server:   server,
+				Nickname: nick,
+				Flags:    flags,
+				Realname: realname,
+			})
+		}
+	case xirc.RPL_WHOSPCRPL:
+		dc, cmd := uc.currentPendingCommand("WHO")
+		if cmd == nil {
+			return fmt.Errorf("unexpected RPL_WHOSPCRPL: no matching pending WHO")
 		} else if dc == nil {
 			return nil
 		}
 
 		dc.SendMessage(msg)
+
+		if len(cmd.Params) > 1 {
+			fields, _ := xirc.ParseWHOXOptions(cmd.Params[1])
+			if strings.IndexByte(fields, 'n') < 0 {
+				return nil
+			}
+			info, err := xirc.ParseWHOXReply(msg, fields)
+			if err != nil {
+				return err
+			}
+			if uc.shouldCacheUserInfo(info.Nickname) {
+				uc.cacheUserInfo(info.Nickname, &upstreamUser{
+					Nickname: info.Nickname,
+					Username: info.Username,
+					Hostname: info.Hostname,
+					Server:   info.Server,
+					Flags:    info.Flags,
+					Account:  info.Account,
+					Realname: info.Realname,
+				})
+			}
+		}
 	case irc.RPL_ENDOFWHO:
 		dc, cmd := uc.dequeueCommand("WHO")
 		if cmd == nil {
@@ -1490,7 +1660,32 @@ func (uc *upstreamConn) handleMessage(ctx context.Context, msg *irc.Message) err
 		uc.forEachDownstreamByID(downstreamID, func(dc *downstreamConn) {
 			dc.SendMessage(msg)
 		})
-	case "AWAY", "ACCOUNT":
+	case "AWAY":
+		// Update user flags, if we already have the flags cached
+		uu := uc.users.Get(msg.Prefix.Name)
+		if uu != nil && uu.Flags != "" {
+			flags := uu.Flags
+			if isAway := len(msg.Params) > 0; isAway {
+				flags = strings.ReplaceAll(flags, "H", "G")
+			} else {
+				flags = strings.ReplaceAll(flags, "G", "H")
+			}
+			uc.cacheUserInfo(msg.Prefix.Name, &upstreamUser{
+				Flags: flags,
+			})
+		}
+
+		uc.forEachDownstream(func(dc *downstreamConn) {
+			dc.SendMessage(msg)
+		})
+	case "ACCOUNT":
+		var account string
+		if err := parseMessageParams(msg, &account); err != nil {
+			return err
+		}
+		uc.cacheUserInfo(msg.Prefix.Name, &upstreamUser{
+			Account: account,
+		})
 		uc.forEachDownstream(func(dc *downstreamConn) {
 			dc.SendMessage(msg)
 		})
@@ -2066,6 +2261,9 @@ func (uc *upstreamConn) updateMonitor() {
 
 	for _, target := range removeList {
 		uc.monitored.Del(target)
+		if !uc.shouldCacheUserInfo(target) {
+			uc.users.Del(target)
+		}
 	}
 }
 
@@ -2132,3 +2330,87 @@ func (uc *upstreamConn) tryRegainNick(nick string) {
 	})
 	uc.pendingRegainNick = wantNick
 }
+
+func (uc *upstreamConn) getCachedWHO(mask, fields string) (l []*upstreamUser, ok bool) {
+	// Non-extended WHO fields
+	if fields == "" {
+		fields = "cuhsnfdr"
+	}
+
+	// Some extensions are required to keep our cached state in sync. We could
+	// require setname for 'r' and chghost for 'h'/'s', but servers usually
+	// implement a QUIT/JOIN fallback, so let's not bother.
+
+	// TODO: Avoid storing fields we cannot keep up to date, instead of storing them
+	//       then failing here. eg if we don't have account-notify, avoid storing the ACCOUNT
+	//       in the first place.
+	if strings.IndexByte(fields, 'a') >= 0 && !uc.caps.IsEnabled("account-notify") {
+		return nil, false
+	}
+	if strings.IndexByte(fields, 'f') >= 0 && !uc.caps.IsEnabled("away-notify") {
+		return nil, false
+	}
+
+	if uu := uc.users.Get(mask); uu != nil {
+		if uu.hasWHOXFields(fields) {
+			return []*upstreamUser{uu}, true
+		}
+	} else if uch := uc.channels.Get(mask); uch != nil {
+		l = make([]*upstreamUser, 0, uch.Members.Len())
+		ok = true
+		uch.Members.ForEach(func(nick string, membershipSet *xirc.MembershipSet) {
+			if !ok {
+				return
+			}
+			uu := uc.users.Get(nick)
+			if uu == nil || !uu.hasWHOXFields(fields) {
+				ok = false
+			} else {
+				l = append(l, uu)
+			}
+		})
+		if !ok {
+			return nil, false
+		}
+		return l, true
+	}
+
+	return nil, false
+}
+
+func (uc *upstreamConn) cacheUserInfo(nick string, info *upstreamUser) {
+	if nick == "" {
+		panic("cacheUserInfo called with empty nickname")
+	}
+
+	uu := uc.users.Get(nick)
+	if uu == nil {
+		if info.Nickname != "" {
+			nick = info.Nickname
+		} else {
+			info.Nickname = nick
+		}
+		uc.users.Set(info)
+	} else {
+		uu.updateFrom(info)
+		if info.Nickname != "" && nick != info.Nickname {
+			uc.users.Del(nick)
+			uc.users.Set(uu)
+		}
+	}
+}
+
+func (uc *upstreamConn) shouldCacheUserInfo(nick string) bool {
+	if uc.isOurNick(nick) {
+		return true
+	}
+	// keep the cached user info only if we MONITOR it, or we share a channel with them
+	if uc.monitored.Has(nick) {
+		return true
+	}
+	found := false
+	uc.channels.ForEach(func(ch *upstreamChannel) {
+		found = found || ch.Members.Has(nick)
+	})
+	return found
+}

user.go

@@ -389,6 +389,7 @@ func (net *network) updateCasemapping(newCasemap casemapping) {
 		uc.channels.ForEach(func(uch *upstreamChannel) {
 			uch.Members.SetCasemapping(newCasemap)
 		})
+		uc.users.SetCasemapping(newCasemap)
 		uc.monitored.SetCasemapping(newCasemap)
 	}
 	net.forEachDownstream(func(dc *downstreamConn) {

xirc/whox.go

@@ -2,6 +2,9 @@ package xirc
 
 import (
 	"gopkg.in/irc.v4"
+
+	"fmt"
+	"strings"
 )
 
 // whoxFields is the list of all WHOX field letters, by order of appearance in
@@ -19,8 +22,8 @@ type WHOXInfo struct {
 	Realname string
 }
 
-func (info *WHOXInfo) get(field byte) string {
+func (info *WHOXInfo) get(k byte) string {
-	switch field {
+	switch k {
 	case 't':
 		return info.Token
 	case 'c':
@@ -55,6 +58,27 @@ func (info *WHOXInfo) get(field byte) string {
 	return ""
 }
 
+func (info *WHOXInfo) set(k byte, v string) {
+	switch k {
+	case 't':
+		info.Token = v
+	case 'u':
+		info.Username = v
+	case 'h':
+		info.Hostname = v
+	case 's':
+		info.Server = v
+	case 'n':
+		info.Nickname = v
+	case 'f':
+		info.Flags = v
+	case 'a':
+		info.Account = v
+	case 'r':
+		info.Realname = v
+	}
+}
+
 func GenerateWHOXReply(prefix *irc.Prefix, nick, fields string, info *WHOXInfo) *irc.Message {
 	if fields == "" {
 		return &irc.Message{
@@ -83,3 +107,46 @@ func GenerateWHOXReply(prefix *irc.Prefix, nick, fields string, info *WHOXInfo)
 		Params:  append([]string{nick}, values...),
 	}
 }
+
+func ParseWHOXOptions(options string) (fields, whoxToken string) {
+	optionsParts := strings.SplitN(options, "%", 2)
+	// TODO: add support for WHOX flags in optionsParts[0]
+	if len(optionsParts) == 2 {
+		optionsParts := strings.SplitN(optionsParts[1], ",", 2)
+		fields = strings.ToLower(optionsParts[0])
+		if len(optionsParts) == 2 && strings.Contains(fields, "t") {
+			whoxToken = optionsParts[1]
+		}
+	}
+	return fields, whoxToken
+}
+
+func ParseWHOXReply(msg *irc.Message, fields string) (*WHOXInfo, error) {
+	if msg.Command != RPL_WHOSPCRPL {
+		return nil, fmt.Errorf("invalid WHOX reply %q", msg.Command)
+	} else if len(msg.Params) == 0 {
+		return nil, fmt.Errorf("invalid RPL_WHOSPCRPL: no params")
+	}
+
+	fieldSet := make(map[byte]bool)
+	for i := 0; i < len(fields); i++ {
+		fieldSet[fields[i]] = true
+	}
+
+	var info WHOXInfo
+	values := msg.Params[1:]
+	for _, field := range whoxFields {
+		if !fieldSet[field] {
+			continue
+		}
+
+		if len(values) == 0 {
+			return nil, fmt.Errorf("invalid RPL_WHOSPCRPL: missing value for field %q", string(field))
+		}
+
+		info.set(field, values[0])
+		values = values[1:]
+	}
+
+	return &info, nil
+}