rsp2k/headscale
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

integration: retry transient docker network ops

Browse source 
Libnetwork endpoint cleanup is eventually consistent. A back-to-back
disconnect+connect on the same network can race teardown and return a
transient error. Wrap the daemon calls in bounded exponential backoff
so TestHASubnetRouterFailoverDockerDisconnect no longer flakes on
phase 4c reconnect.

Fixes #3234
This commit is contained in:
Kristoffer Dalby 2026-04-30 08:18:50 +00:00
parent 3d5c0af4e7
commit 155e42f892
2 changed files with 83 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

38
integration/dockertestutil/network.go
View file

@ -1,11 +1,14 @@
package dockertestutil
import (
"context"
"errors"
"fmt"
"log"
"net"
"time"
"github.com/cenkalti/backoff/v5"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
@ -13,6 +16,20 @@ import (
var ErrContainerNotFound = errors.New("container not found")
// retryDockerOp absorbs eventual-consistency races in libnetwork endpoint cleanup.
func retryDockerOp(ctx context.Context, op func() error) error {
_, err := backoff.Retry(ctx, func() (struct{}, error) {
err := op()
if err != nil {
return struct{}{}, err
}
return struct{}{}, nil
}, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxElapsedTime(30*time.Second))
return err
}
func GetFirstOrCreateNetwork(pool *dockertest.Pool, name string) (*dockertest.Network, error) {
return GetFirstOrCreateNetworkWithSubnet(pool, name, "")
}
@ -72,13 +89,6 @@ func AddContainerToNetwork(
return err
}
err = pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
Container: containers[0].ID,
})
if err != nil {
return err
}
// TODO(kradalby): This doesn't work reliably, but calling the exact same functions
// seem to work fine...
// if container, ok := pool.ContainerByName("/" + testContainer); ok {
@ -88,7 +98,11 @@ func AddContainerToNetwork(
// }
// }
return nil
return retryDockerOp(context.Background(), func() error {
return pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
Container: containers[0].ID,
})
})
}
// DisconnectContainerFromNetwork removes the container from network at
@ -115,9 +129,11 @@ func DisconnectContainerFromNetwork(
return fmt.Errorf("%w: %s", ErrContainerNotFound, testContainer)
}
return pool.Client.DisconnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
Container: containers[0].ID,
Force: true,
return retryDockerOp(context.Background(), func() error {
return pool.Client.DisconnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
Container: containers[0].ID,
Force: true,
})
})
}