Use BARE for internal message IDs

This allows to have shorter and more future-proof IDs. This also
guarantees the IDs will only use reasonable ASCII characters (no
spaces), removing the need to encode them for PING/PONG tokens.

downstream.go

@@ -350,7 +350,7 @@ func (dc *downstreamConn) advanceMessageWithID(msg *irc.Message, id string) {
 
 // ackMsgID acknowledges that a message has been received.
 func (dc *downstreamConn) ackMsgID(id string) {
-	netID, entity, _, err := parseMsgID(id)
+	netID, entity, err := parseMsgID(id, nil)
 	if err != nil {
 		dc.logger.Printf("failed to ACK message ID %q: %v", id, err)
 		return
@@ -365,7 +365,7 @@ func (dc *downstreamConn) ackMsgID(id string) {
 }
 
 func (dc *downstreamConn) sendPing(msgID string) {
-	token := "soju-msgid-" + base64.RawURLEncoding.EncodeToString([]byte(msgID))
+	token := "soju-msgid-" + msgID
 	dc.SendMessage(&irc.Message{
 		Command: "PING",
 		Params:  []string{token},
@@ -377,14 +377,7 @@ func (dc *downstreamConn) handlePong(token string) {
 		dc.logger.Printf("received unrecognized PONG token %q", token)
 		return
 	}
-	token = strings.TrimPrefix(token, "soju-msgid-")
+	msgID := strings.TrimPrefix(token, "soju-msgid-")
-	b, err := base64.RawURLEncoding.DecodeString(token)
-	if err != nil {
-		dc.logger.Printf("received malformed PONG token: %v", err)
-		return
-	}
-	msgID := string(b)
-
 	dc.ackMsgID(msgID)
 }
 

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc
+	git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9
 	github.com/emersion/go-sasl v0.0.0-20200509203442-7bfe0ed36a21
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/klauspost/compress v1.11.7 // indirect

go.sum

@@ -1,6 +1,8 @@
 git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc h1:51BD67xFX+bozd3ZRuOUfalrhx4/nQSh6A9lI08rYOk=
 git.sr.ht/~emersion/go-scfg v0.0.0-20201019143924-142a8aa629fc/go.mod h1:t+Ww6SR24yYnXzEWiNlOY0AFo5E9B73X++10lrSpp4U=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+git.sr.ht/~sircmpwn/getopt v0.0.0-20191230200459-23622cc906b3/go.mod h1:wMEGFFFNuPos7vHmWXfszqImLppbc0wEhh6JBfJIUgw=
+git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9 h1:GpgMhmmlPgaKOSU3WnoaSpZGWgnprcS+ss2w9SchYu4=
+git.sr.ht/~sircmpwn/go-bare v0.0.0-20210331145808-46f9b5e5bcf9/go.mod h1:BVJwbDfVjCjoFiKrhkei6NdGcZYpkDkdyCdg1ukytRA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -36,7 +38,6 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
-github.com/klauspost/compress v1.10.3 h1:OP96hzwJVBIHYU52pVTI6CczrxPvrGfgqF9N5eTO0Q8=
 github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg=
 github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
@@ -56,8 +57,9 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
@@ -72,7 +74,6 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78 h1:nVuTkr9L6Bq62qpUqKo/RnZCFfzDBL0bYo6w9OJUqZY=
 golang.org/x/sys v0.0.0-20210113181707-4bcb84eeeb78/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -89,5 +90,7 @@ gopkg.in/irc.v3 v3.1.4/go.mod h1:shO2gz8+PVeS+4E6GAny88Z0YVVQSxQghdrMVGQsR9s=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 nhooyr.io/websocket v1.8.6 h1:s+C3xAMLwGmlI31Nyn/eAehUlZPwfYZu2JXM621Q5/k=
 nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=

msgstore.go

@@ -1,11 +1,12 @@
 package soju
 
 import (
+	"bytes"
+	"encoding/base64"
 	"fmt"
-	"strconv"
-	"strings"
 	"time"
 
+	"git.sr.ht/~sircmpwn/go-bare"
 	"gopkg.in/irc.v3"
 )
 
@@ -29,18 +30,73 @@ type chatHistoryMessageStore interface {
 	LoadAfterTime(network *network, entity string, t time.Time, limit int) ([]*irc.Message, error)
 }
 
-func formatMsgID(netID int64, entity, extra string) string {
+type msgIDType uint
-	return fmt.Sprintf("%v %v %v", netID, entity, extra)
+
+const (
+	msgIDNone msgIDType = iota
+	msgIDMemory
+	msgIDFS
+)
+
+const msgIDVersion uint = 0
+
+type msgIDHeader struct {
+	Version uint
+	Network bare.Int
+	Target  string
+	Type    msgIDType
 }
 
-func parseMsgID(s string) (netID int64, entity, extra string, err error) {
+type msgIDBody interface {
-	l := strings.SplitN(s, " ", 3)
+	msgIDType() msgIDType
-	if len(l) != 3 {
-		return 0, "", "", fmt.Errorf("invalid message ID %q: expected 3 fields", s)
 }
-	netID, err = strconv.ParseInt(l[0], 10, 64)
+
+func formatMsgID(netID int64, target string, body msgIDBody) string {
+	var buf bytes.Buffer
+	w := bare.NewWriter(&buf)
+
+	header := msgIDHeader{
+		Version: msgIDVersion,
+		Network: bare.Int(netID),
+		Target:  target,
+		Type:    body.msgIDType(),
+	}
+	if err := bare.MarshalWriter(w, &header); err != nil {
+		panic(err)
+	}
+	if err := bare.MarshalWriter(w, body); err != nil {
+		panic(err)
+	}
+	return base64.RawURLEncoding.EncodeToString(buf.Bytes())
+}
+
+func parseMsgID(s string, body msgIDBody) (netID int64, target string, err error) {
+	b, err := base64.RawURLEncoding.DecodeString(s)
 	if err != nil {
-		return 0, "", "", fmt.Errorf("invalid message ID %q: %v", s, err)
+		return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
 	}
-	return netID, l[1], l[2], nil
+
+	r := bare.NewReader(bytes.NewReader(b))
+
+	var header msgIDHeader
+	if err := bare.UnmarshalBareReader(r, &header); err != nil {
+		return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
+	}
+
+	if header.Version != msgIDVersion {
+		return 0, "", fmt.Errorf("invalid internal message ID: got version %v, want %v", header.Version, msgIDVersion)
+	}
+
+	if body != nil {
+		typ := body.msgIDType()
+		if header.Type != typ {
+			return 0, "", fmt.Errorf("invalid internal message ID: got type %v, want %v", header.Type, typ)
+		}
+
+		if err := bare.UnmarshalBareReader(r, body); err != nil {
+			return 0, "", fmt.Errorf("invalid internal message ID: %v", err)
+		}
+	}
+
+	return int64(header.Network), header.Target, nil
 }

msgstore_fs.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"git.sr.ht/~sircmpwn/go-bare"
 	"gopkg.in/irc.v3"
 )
 
@@ -16,6 +17,45 @@ const fsMessageStoreMaxTries = 100
 
 var escapeFilename = strings.NewReplacer("/", "-", "\\", "-")
 
+type date struct {
+	Year, Month, Day int
+}
+
+func newDate(t time.Time) date {
+	year, month, day := t.Date()
+	return date{year, int(month), day}
+}
+
+func (d date) Time() time.Time {
+	return time.Date(d.Year, time.Month(d.Month), d.Day, 0, 0, 0, 0, time.Local)
+}
+
+type fsMsgID struct {
+	Date   date
+	Offset bare.Int
+}
+
+func (fsMsgID) msgIDType() msgIDType {
+	return msgIDFS
+}
+
+func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) {
+	var id fsMsgID
+	netID, entity, err = parseMsgID(s, &id)
+	if err != nil {
+		return 0, "", time.Time{}, 0, err
+	}
+	return netID, entity, id.Date.Time(), int64(id.Offset), nil
+}
+
+func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string {
+	id := fsMsgID{
+		Date:   newDate(t),
+		Offset: bare.Int(offset),
+	}
+	return formatMsgID(netID, entity, &id)
+}
+
 // fsMessageStore is a per-user on-disk store for IRC messages.
 type fsMessageStore struct {
 	root string
@@ -36,27 +76,6 @@ func (ms *fsMessageStore) logPath(network *network, entity string, t time.Time)
 	return filepath.Join(ms.root, escapeFilename.Replace(network.GetName()), escapeFilename.Replace(entity), filename)
 }
 
-func parseFSMsgID(s string) (netID int64, entity string, t time.Time, offset int64, err error) {
-	netID, entity, extra, err := parseMsgID(s)
-	if err != nil {
-		return 0, "", time.Time{}, 0, err
-	}
-
-	var year, month, day int
-	_, err = fmt.Sscanf(extra, "%04d-%02d-%02d %d", &year, &month, &day, &offset)
-	if err != nil {
-		return 0, "", time.Time{}, 0, fmt.Errorf("invalid message ID %q: %v", s, err)
-	}
-	t = time.Date(year, time.Month(month), day, 0, 0, 0, 0, time.Local)
-	return netID, entity, t, offset, nil
-}
-
-func formatFSMsgID(netID int64, entity string, t time.Time, offset int64) string {
-	year, month, day := t.Date()
-	extra := fmt.Sprintf("%04d-%02d-%02d %d", year, month, day, offset)
-	return formatMsgID(netID, entity, extra)
-}
-
 // nextMsgID queries the message ID for the next message to be written to f.
 func nextFSMsgID(network *network, entity string, t time.Time, f *os.File) (string, error) {
 	offset, err := f.Seek(0, io.SeekEnd)

msgstore_memory.go

@@ -2,29 +2,34 @@ package soju
 
 import (
 	"fmt"
-	"strconv"
 	"time"
 
+	"git.sr.ht/~sircmpwn/go-bare"
 	"gopkg.in/irc.v3"
 )
 
 const messageRingBufferCap = 4096
 
+type memoryMsgID struct {
+	Seq bare.Uint
+}
+
+func (memoryMsgID) msgIDType() msgIDType {
+	return msgIDMemory
+}
+
 func parseMemoryMsgID(s string) (netID int64, entity string, seq uint64, err error) {
-	netID, entity, extra, err := parseMsgID(s)
+	var id memoryMsgID
+	netID, entity, err = parseMsgID(s, &id)
 	if err != nil {
 		return 0, "", 0, err
 	}
-	seq, err = strconv.ParseUint(extra, 10, 64)
+	return netID, entity, uint64(id.Seq), nil
-	if err != nil {
-		return 0, "", 0, fmt.Errorf("failed to parse message ID %q: %v", s, err)
-	}
-	return netID, entity, seq, nil
 }
 
 func formatMemoryMsgID(netID int64, entity string, seq uint64) string {
-	extra := strconv.FormatUint(seq, 10)
+	id := memoryMsgID{bare.Uint(seq)}
-	return formatMsgID(netID, entity, extra)
+	return formatMsgID(netID, entity, &id)
 }
 
 type ringBufferKey struct {