Add exponential backoff when re-connecting to upstream
The first reconnection attempt waits for 1min, the second the 2min, and so on up to 10min. There's a 1min jitter so that multiple failed connections don't try to reconnect at the exact same time. Closes: https://todo.sr.ht/~emersion/soju/161
This commit is contained in:
parent
8acc07d591
commit
578020e553
40
rate.go
Normal file
40
rate.go
Normal file
@ -0,0 +1,40 @@
|
||||
package soju
|
||||
|
||||
import (
|
||||
"math/rand"
|
||||
"time"
|
||||
)
|
||||
|
||||
// backoffer implements a simple exponential backoff.
|
||||
type backoffer struct {
|
||||
min, max, jitter time.Duration
|
||||
n int64
|
||||
}
|
||||
|
||||
func newBackoffer(min, max, jitter time.Duration) *backoffer {
|
||||
return &backoffer{min: min, max: max, jitter: jitter}
|
||||
}
|
||||
|
||||
func (b *backoffer) Reset() {
|
||||
b.n = 0
|
||||
}
|
||||
|
||||
func (b *backoffer) Next() time.Duration {
|
||||
if b.n == 0 {
|
||||
b.n = 1
|
||||
return 0
|
||||
}
|
||||
|
||||
d := time.Duration(b.n) * b.min
|
||||
if d > b.max {
|
||||
d = b.max
|
||||
} else {
|
||||
b.n *= 2
|
||||
}
|
||||
|
||||
if b.jitter != 0 {
|
||||
d += time.Duration(rand.Int63n(int64(b.jitter)))
|
||||
}
|
||||
|
||||
return d
|
||||
}
|
@ -23,7 +23,9 @@ import (
|
||||
)
|
||||
|
||||
// TODO: make configurable
|
||||
var retryConnectDelay = time.Minute
|
||||
var retryConnectMinDelay = time.Minute
|
||||
var retryConnectMaxDelay = 10 * time.Minute
|
||||
var retryConnectJitter = time.Minute
|
||||
var connectTimeout = 15 * time.Second
|
||||
var writeTimeout = 10 * time.Second
|
||||
var upstreamMessageDelay = 2 * time.Second
|
||||
|
6
user.go
6
user.go
@ -190,13 +190,14 @@ func (net *network) run() {
|
||||
}
|
||||
|
||||
var lastTry time.Time
|
||||
backoff := newBackoffer(retryConnectMinDelay, retryConnectMaxDelay, retryConnectJitter)
|
||||
for {
|
||||
if net.isStopped() {
|
||||
return
|
||||
}
|
||||
|
||||
if dur := time.Now().Sub(lastTry); dur < retryConnectDelay {
|
||||
delay := retryConnectDelay - dur
|
||||
delay := backoff.Next() - time.Now().Sub(lastTry)
|
||||
if delay > 0 {
|
||||
net.logger.Printf("waiting %v before trying to reconnect to %q", delay.Truncate(time.Second), net.Addr)
|
||||
time.Sleep(delay)
|
||||
}
|
||||
@ -247,6 +248,7 @@ func (net *network) run() {
|
||||
}
|
||||
|
||||
net.user.srv.metrics.upstreams.Add(-1)
|
||||
backoff.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user