state: delete routes package, port primary route tests

Remove hscontrol/routes/. Port the named scenarios and the rapid
property test to hscontrol/state/.

Updates #3203
This commit is contained in:
Kristoffer Dalby 2026-04-28 12:16:34 +00:00
parent 437754aeea
commit 66ac785c22
6 changed files with 652 additions and 1179 deletions

View file

@ -0,0 +1,331 @@
package state
import (
"fmt"
"net/netip"
"slices"
"testing"
"github.com/juanfont/headscale/hscontrol/types"
"pgregory.net/rapid"
"tailscale.com/tailcfg"
"tailscale.com/types/key"
)
// model mirrors the expected primary-route assignment given the
// operations applied so far. It is intentionally simple — close to
// what computePrimaries (node_store.go) prescribes — so divergence
// between the model and the snapshot's primaries map flags a bug in
// the algorithm.
//
// The model lives in the test file because the algorithm under test
// is the one inside snapshotFromNodes; the model's job is to predict
// the same answer from a separate, deliberately direct implementation.
type primariesModel struct {
connected map[types.NodeID]bool
prefixes map[types.NodeID][]netip.Prefix
unhealthy map[types.NodeID]bool
// primary[p] is the current primary for prefix p. The
// implementation preserves the current primary across changes to
// avoid flapping, so the model has to track this across
// operations rather than recompute a fresh choice each time.
primary map[netip.Prefix]types.NodeID
}
func newPrimariesModel() *primariesModel {
return &primariesModel{
connected: map[types.NodeID]bool{},
prefixes: map[types.NodeID][]netip.Prefix{},
unhealthy: map[types.NodeID]bool{},
primary: map[netip.Prefix]types.NodeID{},
}
}
// advertisersByPrefix returns the connected nodes that announce each
// prefix, sorted by NodeID (matches computePrimaries' iteration).
func (m *primariesModel) advertisersByPrefix() map[netip.Prefix][]types.NodeID {
out := map[netip.Prefix][]types.NodeID{}
for n, prefs := range m.prefixes {
if !m.connected[n] {
continue
}
for _, p := range prefs {
out[p] = append(out[p], n)
}
}
for _, nodes := range out {
slices.Sort(nodes)
}
return out
}
// updatePrimaries reapplies the algorithm to recompute the primary
// for each prefix. Called after every operation.
func (m *primariesModel) updatePrimaries() {
advertisers := m.advertisersByPrefix()
// Drop primaries for prefixes that no longer have any advertiser.
for p := range m.primary {
if _, ok := advertisers[p]; !ok {
delete(m.primary, p)
}
}
for p, nodes := range advertisers {
if cur, ok := m.primary[p]; ok {
if slices.Contains(nodes, cur) && !m.unhealthy[cur] {
continue
}
}
var (
selected types.NodeID
found bool
)
for _, n := range nodes {
if !m.unhealthy[n] {
selected = n
found = true
break
}
}
if !found && len(nodes) >= 1 {
selected = nodes[0]
found = true
}
if found {
m.primary[p] = selected
}
}
}
// allPrefixes returns every prefix mentioned by any connected node.
func (m *primariesModel) allPrefixes() []netip.Prefix {
seen := map[netip.Prefix]bool{}
for n, prefs := range m.prefixes {
if !m.connected[n] {
continue
}
for _, p := range prefs {
seen[p] = true
}
}
out := make([]netip.Prefix, 0, len(seen))
for p := range seen {
out = append(out, p)
}
return out
}
func samePrefixSet(a, b []netip.Prefix) bool {
if len(a) != len(b) {
return false
}
aa := slices.Clone(a)
bb := slices.Clone(b)
slices.SortFunc(aa, netip.Prefix.Compare)
slices.SortFunc(bb, netip.Prefix.Compare)
return slices.Equal(aa, bb)
}
// checkPrimariesProperties asserts every rule we expect of the snapshot's
// primaries map given the model.
func checkPrimariesProperties(rt *rapid.T, ns *NodeStore, m *primariesModel, nodeIDs []types.NodeID) {
rt.Helper()
expectedByNode := map[types.NodeID][]netip.Prefix{}
for p, owner := range m.primary {
expectedByNode[owner] = append(expectedByNode[owner], p)
}
for _, id := range nodeIDs {
got := ns.PrimaryRoutesForNode(id)
want := expectedByNode[id]
if !samePrefixSet(got, want) {
rt.Fatalf(
"PrimaryRoutesForNode(%d) = %v, model expected %v",
id, got, want,
)
}
if want := !m.unhealthy[id]; ns.IsNodeHealthy(id) != want {
rt.Fatalf(
"IsNodeHealthy(%d) = %v, want %v",
id, ns.IsNodeHealthy(id), want,
)
}
}
// Every prefix that has at least one connected advertiser must
// have a primary in the snapshot. Issue #3203 manifests as a
// prefix silently losing its primary after a disconnect/reconnect
// cycle.
for _, p := range m.allPrefixes() {
want, expectExists := m.primary[p]
if !expectExists {
continue
}
got, ok := ns.PrimaryRouteFor(p)
if !ok {
rt.Fatalf(
"prefix %s has at least one advertiser in the model but no primary in NodeStore",
p,
)
}
if want != got {
rt.Fatalf(
"prefix %s: snapshot primary = %d, model expected %d",
p, got, want,
)
}
}
}
// nodeForRapid builds a minimal types.Node for use in property
// tests. Tests drive (IsOnline, Hostinfo.RoutableIPs, ApprovedRoutes,
// Unhealthy) via UpdateNode; the rest stays fixed.
func nodeForRapid(id types.NodeID) types.Node {
mk := key.NewMachine()
nk := key.NewNode()
return types.Node{
ID: id,
Hostname: fmt.Sprintf("rapid-%d", id),
MachineKey: mk.Public(),
NodeKey: nk.Public(),
UserID: new(uint(1)),
User: &types.User{Name: "rapid"},
IsOnline: new(false),
Hostinfo: &tailcfg.Hostinfo{},
}
}
// TestPrimaryRoutesProperty drives NodeStore with a randomised
// sequence of high-level operations and checks that the snapshot's
// primaries map matches a reference model after every step.
//
// Background: issue #3203 reports that HA tracking enters a stuck
// state after a sequence of disconnect/reconnect events. The narrow
// integration and servertest reproductions written for the bug do
// not fail on upstream/main, so this property test broadens the
// search by letting rapid generate sequences we have not enumerated
// by hand.
func TestPrimaryRoutesProperty(t *testing.T) {
rapid.Check(t, func(rt *rapid.T) {
const numNodes = 4
nodeIDs := make([]types.NodeID, 0, numNodes)
for i := 1; i <= numNodes; i++ {
nodeIDs = append(nodeIDs, types.NodeID(i))
}
prefixes := []netip.Prefix{
netip.MustParsePrefix("10.0.0.0/24"),
netip.MustParsePrefix("10.0.1.0/24"),
}
ns := NewNodeStore(nil, allowAllPeersFunc, TestBatchSize, TestBatchTimeout)
ns.Start()
defer ns.Stop()
for _, id := range nodeIDs {
ns.PutNode(nodeForRapid(id))
}
m := newPrimariesModel()
nodeGen := rapid.SampledFrom(nodeIDs)
prefixSubsetGen := rapid.SliceOfNDistinct(
rapid.SampledFrom(prefixes),
0, len(prefixes),
func(p netip.Prefix) string { return p.String() },
)
opCount := rapid.IntRange(5, 60).Draw(rt, "opCount")
for step := range opCount {
op := rapid.IntRange(0, 4).Draw(rt, fmt.Sprintf("op_%d", step))
id := nodeGen.Draw(rt, fmt.Sprintf("id_%d", step))
switch op {
case 0: // ConnectAdvertise — Connect path clears Unhealthy.
prefs := prefixSubsetGen.Draw(rt, fmt.Sprintf("prefs_%d", step))
ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(true)
n.Unhealthy = false
n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: prefs}
n.ApprovedRoutes = prefs
})
if len(prefs) == 0 {
delete(m.connected, id)
delete(m.prefixes, id)
} else {
m.connected[id] = true
m.prefixes[id] = prefs
}
delete(m.unhealthy, id)
case 1: // Disconnect — IsOnline=false; ApprovedRoutes persists.
ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(false)
})
delete(m.connected, id)
case 2: // ProbeUnhealthy — HA prober marks node bad.
ns.UpdateNode(id, func(n *types.Node) {
n.Unhealthy = true
})
m.unhealthy[id] = true
case 3: // ProbeHealthy — HA prober marks node good.
ns.UpdateNode(id, func(n *types.Node) {
n.Unhealthy = false
})
delete(m.unhealthy, id)
case 4: // ApprovedRoutesChange — change advertised prefs without touching health.
prefs := prefixSubsetGen.Draw(rt, fmt.Sprintf("prefs_%d", step))
ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(true)
n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: prefs}
n.ApprovedRoutes = prefs
})
if len(prefs) == 0 {
delete(m.connected, id)
delete(m.prefixes, id)
} else {
m.connected[id] = true
m.prefixes[id] = prefs
}
}
m.updatePrimaries()
checkPrimariesProperties(rt, ns, m, nodeIDs)
}
})
}

View file

@ -0,0 +1,319 @@
package state
import (
"net/netip"
"slices"
"testing"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"tailscale.com/tailcfg"
)
// mp wraps netip.MustParsePrefix.
func mp(prefix string) netip.Prefix {
return netip.MustParsePrefix(prefix)
}
// primariesFixture builds a NodeStore with the requested node IDs
// pre-registered (offline, no routes) and provides terse helpers for
// driving the kinds of state transitions the algorithm cares about.
type primariesFixture struct {
t *testing.T
ns *NodeStore
}
func newPrimariesFixture(t *testing.T, ids ...types.NodeID) *primariesFixture {
t.Helper()
ns := NewNodeStore(nil, allowAllPeersFunc, TestBatchSize, TestBatchTimeout)
ns.Start()
t.Cleanup(ns.Stop)
for _, id := range ids {
ns.PutNode(nodeForRapid(id))
}
return &primariesFixture{t: t, ns: ns}
}
// advertise mirrors State.Connect: marks the node online, clears
// Unhealthy, and sets approved + announced routes to prefs. An empty
// prefs argument leaves the node online but advertising nothing.
func (f *primariesFixture) advertise(id types.NodeID, prefs ...netip.Prefix) {
f.t.Helper()
f.ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(true)
n.Unhealthy = false
n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: prefs}
n.ApprovedRoutes = prefs
})
}
// approveRoutes mirrors State.SetApprovedRoutes / Hostinfo updates:
// it changes the node's announced + approved set without touching
// Unhealthy.
func (f *primariesFixture) approveRoutes(id types.NodeID, prefs ...netip.Prefix) {
f.t.Helper()
f.ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(true)
n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: prefs}
n.ApprovedRoutes = prefs
})
}
// disconnect mirrors State.Disconnect: marks the node offline. The
// snapshot rebuild treats an offline node as a non-advertiser.
func (f *primariesFixture) disconnect(id types.NodeID) {
f.t.Helper()
f.ns.UpdateNode(id, func(n *types.Node) {
n.IsOnline = new(false)
})
}
// unhealthy mirrors State.SetNodeUnhealthy(id, true).
func (f *primariesFixture) unhealthy(id types.NodeID) {
f.t.Helper()
f.ns.UpdateNode(id, func(n *types.Node) {
n.Unhealthy = true
})
}
// healthy mirrors State.SetNodeUnhealthy(id, false).
func (f *primariesFixture) healthy(id types.NodeID) {
f.t.Helper()
f.ns.UpdateNode(id, func(n *types.Node) {
n.Unhealthy = false
})
}
// requirePrimary asserts that prefix has node id as its primary.
func (f *primariesFixture) requirePrimary(prefix netip.Prefix, id types.NodeID) {
f.t.Helper()
got, ok := f.ns.PrimaryRouteFor(prefix)
require.True(f.t, ok, "expected a primary for %s, got none", prefix)
require.Equal(f.t, id, got, "primary for %s", prefix)
}
// requireNoPrimary asserts that prefix has no primary at all.
func (f *primariesFixture) requireNoPrimary(prefix netip.Prefix) {
f.t.Helper()
_, ok := f.ns.PrimaryRouteFor(prefix)
require.False(f.t, ok, "expected no primary for %s", prefix)
}
// requireNodeRoutes asserts the set of prefixes for which id is the
// primary, regardless of order.
func (f *primariesFixture) requireNodeRoutes(id types.NodeID, want ...netip.Prefix) {
f.t.Helper()
got := f.ns.PrimaryRoutesForNode(id)
gotSorted := slices.Clone(got)
wantSorted := slices.Clone(want)
slices.SortFunc(gotSorted, netip.Prefix.Compare)
slices.SortFunc(wantSorted, netip.Prefix.Compare)
require.Equal(f.t, wantSorted, gotSorted, "primary routes for node %d", id)
}
func TestPrimaries_SingleNodeSingleRoute(t *testing.T) {
f := newPrimariesFixture(t, 1)
f.advertise(1, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.requireNodeRoutes(1, mp("192.168.1.0/24"))
}
func TestPrimaries_TwoNodesDifferentRoutes(t *testing.T) {
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.2.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.requirePrimary(mp("192.168.2.0/24"), 2)
}
func TestPrimaries_OverlappingRoutesLowerIDWins(t *testing.T) {
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.requireNodeRoutes(1, mp("192.168.1.0/24"))
f.requireNodeRoutes(2)
}
func TestPrimaries_AntiFlapPreservesCurrentPrimary(t *testing.T) {
// A primary that disappears (advertiser leaves the set) should
// trigger failover. When the original primary returns, the new
// primary keeps the assignment — anti-flap.
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.disconnect(1)
f.requirePrimary(mp("192.168.1.0/24"), 2)
f.advertise(1, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 2)
}
func TestPrimaries_ClearRoutesDropsPrimary(t *testing.T) {
f := newPrimariesFixture(t, 1)
f.advertise(1, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.approveRoutes(1)
f.requireNoPrimary(mp("192.168.1.0/24"))
}
func TestPrimaries_DisconnectDropsLastAdvertiserPrimary(t *testing.T) {
f := newPrimariesFixture(t, 1)
f.advertise(1, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.disconnect(1)
f.requireNoPrimary(mp("192.168.1.0/24"))
}
func TestPrimaries_UnhealthyTriggersFailover(t *testing.T) {
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.requirePrimary(mp("192.168.1.0/24"), 1)
f.unhealthy(1)
f.requirePrimary(mp("192.168.1.0/24"), 2)
}
func TestPrimaries_RecoveryFromUnhealthyNoFlap(t *testing.T) {
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.unhealthy(1)
f.requirePrimary(mp("192.168.1.0/24"), 2)
f.healthy(1)
f.requirePrimary(mp("192.168.1.0/24"), 2)
}
func TestPrimaries_AllUnhealthyKeepsLowestIDPrimary(t *testing.T) {
// When every advertiser is unhealthy the algorithm degrades to
// the lowest-ID advertiser rather than going dark — peers should
// still see *some* primary so connectivity can recover when one
// of them flips healthy.
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.unhealthy(1)
f.unhealthy(2)
f.requirePrimary(mp("192.168.1.0/24"), 1)
}
func TestPrimaries_ExitRouteNotElected(t *testing.T) {
// Exit routes (0.0.0.0/0, ::/0) are not subject to HA primary
// election — every approved exit-route advertiser keeps it.
f := newPrimariesFixture(t, 1)
exitV4 := mp("0.0.0.0/0")
f.advertise(1, exitV4)
f.requireNoPrimary(exitV4)
}
func TestPrimaries_RegressionIssue3203_BothOfflineThenOneReturns(t *testing.T) {
// Issue #3203: with two HA advertisers, dropping both then
// bringing one back used to leave the prefix without any
// primary. After the refactor the snapshot recomputes primaries
// on every NodeStore write, so the returning advertiser must
// be elected.
prefix := mp("10.0.0.0/24")
f := newPrimariesFixture(t, 1, 2)
f.advertise(1, prefix)
f.advertise(2, prefix)
f.requirePrimary(prefix, 1)
f.disconnect(1)
f.requirePrimary(prefix, 2)
f.disconnect(2)
f.requireNoPrimary(prefix)
f.advertise(2, prefix)
f.requirePrimary(prefix, 2)
}
func TestPrimaries_HANodes(t *testing.T) {
tests := []struct {
name string
setup func(*primariesFixture)
want map[netip.Prefix][]types.NodeID
}{
{
name: "single-node-not-ha",
setup: func(f *primariesFixture) {
f.advertise(1, mp("192.168.1.0/24"))
},
want: map[netip.Prefix][]types.NodeID{},
},
{
name: "two-nodes-same-prefix-is-ha",
setup: func(f *primariesFixture) {
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
},
want: map[netip.Prefix][]types.NodeID{
mp("192.168.1.0/24"): {1, 2},
},
},
{
name: "two-nodes-different-prefixes-not-ha",
setup: func(f *primariesFixture) {
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.2.0/24"))
},
want: map[netip.Prefix][]types.NodeID{},
},
{
name: "three-nodes-two-share-prefix",
setup: func(f *primariesFixture) {
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.advertise(3, mp("10.0.0.0/8"))
},
want: map[netip.Prefix][]types.NodeID{
mp("192.168.1.0/24"): {1, 2},
},
},
{
name: "three-nodes-all-share",
setup: func(f *primariesFixture) {
f.advertise(1, mp("192.168.1.0/24"))
f.advertise(2, mp("192.168.1.0/24"))
f.advertise(3, mp("192.168.1.0/24"))
},
want: map[netip.Prefix][]types.NodeID{
mp("192.168.1.0/24"): {1, 2, 3},
},
},
{
name: "empty",
setup: func(*primariesFixture) {
},
want: map[netip.Prefix][]types.NodeID{},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f := newPrimariesFixture(t, 1, 2, 3)
tt.setup(f)
got := f.ns.HANodes()
assert.Equal(t, tt.want, got)
})
}
}