state: defer HA failover when probe target reconnected mid-cycle

The HA prober dispatches a PingRequest, waits ProbeTimeout (5s), and
marks the node unhealthy if no callback arrives. A node that bounced
its poll session between probe cycles satisfies two conditions that
conspire to fail TestHASubnetRouterFailover: a probe queued against
the previous session is silently dropped when the worker writes to
the closed connection (timeout always fires), and a probe sent
immediately after reconnect lands while wgengine is still rebuilding
magicsock state from the new netmap. Either path installs a spurious
unhealthy bit, which sends the preserved-primary anti-flap the wrong
way.

Record the session observed at dispatch time and drop the timeout
path if the node reconnected since. Require the session to survive
a full probe cycle before a timeout can drive a failover.
This commit is contained in:
Kristoffer Dalby 2026-05-13 13:20:22 +00:00
parent a345a22a3b
commit fb8eecae25
2 changed files with 156 additions and 11 deletions

View file

@ -283,6 +283,77 @@ func TestHAHealthProbe_SetUnhealthyNoRoutesIsNoOp(t *testing.T) {
"SetNodeUnhealthy on node with no approved routes should be a no-op")
}
// TestHAHealthProbe_ReconnectDuringProbeKeepsHealthy reproduces the
// race that surfaced as a TestHASubnetRouterFailover flake: a probe
// dispatched against the previous poll session sees the timeout fire
// while the client is briefly disconnected. With the session guard in
// [HAHealthProber.ProbeOnce], the timeout path observes the reconnect
// and bails out instead of installing a spurious Unhealthy bit.
//
// Without the guard, the primary fails over to the standby and the
// anti-flap election preserves that choice even after the original
// primary is fully back online.
func TestHAHealthProbe_ReconnectDuringProbeKeepsHealthy(t *testing.T) {
t.Parallel()
srv := servertest.NewServer(t)
user := srv.CreateUser(t, "ha-probe-reconnect")
route := netip.MustParsePrefix("10.102.0.0/24")
c1 := servertest.NewClient(t, srv, "ha-pr-r1", servertest.WithUser(user))
c2 := servertest.NewClient(t, srv, "ha-pr-r2", servertest.WithUser(user))
c1.WaitForPeers(t, 1, 10*time.Second)
c2.WaitForPeers(t, 1, 10*time.Second)
nodeID1 := advertiseAndApproveRoute(t, srv, c1, route)
advertiseAndApproveRoute(t, srv, c2, route)
// Node 1 is primary (lowest ID, healthy).
require.Contains(t,
srv.State().GetNodePrimaryRoutes(nodeID1), route,
"node 1 should be primary initially")
prober := state.NewHAHealthProber(
srv.State(),
types.HARouteConfig{
ProbeInterval: 30 * time.Second,
ProbeTimeout: 2 * time.Second,
},
srv.URL,
srv.App.MapBatcher().IsConnected,
)
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
// TestClient does not implement ping responses, so every probe
// times out. We exploit that to observe the timeout path under a
// reconnect race: kick a probe in a goroutine, bounce the
// primary's poll session, and confirm the prober drops the stale
// timeout instead of marking the node unhealthy.
done := make(chan struct{})
go func() {
defer close(done)
prober.ProbeOnce(ctx, srv.App.Change)
}()
c1.Disconnect(t)
c1.Reconnect(t)
c1.WaitForPeers(t, 1, 10*time.Second)
<-done
assert.True(t, srv.State().IsNodeHealthy(nodeID1),
"reconnect during probe must not flip node unhealthy")
assert.Contains(t,
srv.State().GetNodePrimaryRoutes(nodeID1), route,
"node 1 should remain primary after stale-probe timeout")
}
// TestHAHealthProbe_NoHARoutes verifies that the prober is a no-op
// when no HA configuration exists.
func TestHAHealthProbe_NoHARoutes(t *testing.T) {