Add exponential backoff when re-connecting to upstream

The first reconnection attempt waits for 1min, the second the 2min,
and so on up to 10min. There's a 1min jitter so that multiple failed
connections don't try to reconnect at the exact same time.

Closes: https://todo.sr.ht/~emersion/soju/161

rate.go

@@ -0,0 +1,40 @@
+package soju
+
+import (
+	"math/rand"
+	"time"
+)
+
+// backoffer implements a simple exponential backoff.
+type backoffer struct {
+	min, max, jitter time.Duration
+	n                int64
+}
+
+func newBackoffer(min, max, jitter time.Duration) *backoffer {
+	return &backoffer{min: min, max: max, jitter: jitter}
+}
+
+func (b *backoffer) Reset() {
+	b.n = 0
+}
+
+func (b *backoffer) Next() time.Duration {
+	if b.n == 0 {
+		b.n = 1
+		return 0
+	}
+
+	d := time.Duration(b.n) * b.min
+	if d > b.max {
+		d = b.max
+	} else {
+		b.n *= 2
+	}
+
+	if b.jitter != 0 {
+		d += time.Duration(rand.Int63n(int64(b.jitter)))
+	}
+
+	return d
+}

server.go

@@ -23,7 +23,9 @@ import (
 )
 
 // TODO: make configurable
-var retryConnectDelay = time.Minute
+var retryConnectMinDelay = time.Minute
+var retryConnectMaxDelay = 10 * time.Minute
+var retryConnectJitter = time.Minute
 var connectTimeout = 15 * time.Second
 var writeTimeout = 10 * time.Second
 var upstreamMessageDelay = 2 * time.Second

user.go

@@ -190,13 +190,14 @@ func (net *network) run() {
 	}
 
 	var lastTry time.Time
+	backoff := newBackoffer(retryConnectMinDelay, retryConnectMaxDelay, retryConnectJitter)
 	for {
 		if net.isStopped() {
 			return
 		}
 
-		if dur := time.Now().Sub(lastTry); dur < retryConnectDelay {
+		delay := backoff.Next() - time.Now().Sub(lastTry)
-			delay := retryConnectDelay - dur
+		if delay > 0 {
 			net.logger.Printf("waiting %v before trying to reconnect to %q", delay.Truncate(time.Second), net.Addr)
 			time.Sleep(delay)
 		}
@@ -247,6 +248,7 @@ func (net *network) run() {
 		}
 
 		net.user.srv.metrics.upstreams.Add(-1)
+		backoff.Reset()
 	}
 }