integration: retry transient docker network ops
Libnetwork endpoint cleanup is eventually consistent. A back-to-back disconnect+connect on the same network can race teardown and return a transient error. Wrap the daemon calls in bounded exponential backoff so TestHASubnetRouterFailoverDockerDisconnect no longer flakes on phase 4c reconnect. Fixes #3234
This commit is contained in:
parent
3d5c0af4e7
commit
155e42f892
2 changed files with 83 additions and 11 deletions
56
integration/dockertestutil/network_test.go
Normal file
56
integration/dockertestutil/network_test.go
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
package dockertestutil
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
errTransientEndpointExists = errors.New("endpoint with name foo already exists in network bar")
|
||||
errPermanent = errors.New("permanent error")
|
||||
)
|
||||
|
||||
func TestRetryDockerOp_RecoversFromTransient(t *testing.T) {
|
||||
var attempts atomic.Int32
|
||||
|
||||
op := func() error {
|
||||
if attempts.Add(1) < 3 {
|
||||
return errTransientEndpointExists
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
err := retryDockerOp(context.Background(), op)
|
||||
if err != nil {
|
||||
t.Fatalf("retryDockerOp should recover from 2 transient errors, got: %v", err)
|
||||
}
|
||||
|
||||
if got := attempts.Load(); got != 3 {
|
||||
t.Fatalf("expected 3 attempts, got %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryDockerOp_RespectsContextCancellation(t *testing.T) {
|
||||
op := func() error {
|
||||
return errPermanent
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
|
||||
defer cancel()
|
||||
|
||||
start := time.Now()
|
||||
err := retryDockerOp(ctx, op)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("retryDockerOp should fail when op always errors")
|
||||
}
|
||||
|
||||
if elapsed > 5*time.Second {
|
||||
t.Fatalf("retryDockerOp should honour ctx deadline (~200ms), took %s", elapsed)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue