integration: harden ACL test ergonomics

tsic.Curl returned ("", nil) when curl exited 0 with a zero-byte body —
the usual signature of a mid-stream reset — so EventuallyWithT could
not retry. Return an error on empty body instead.

Replace the 56 inline curl + assert.Len(13) blocks with
assertCurlDockerHostname so the empty-body fix benefits every callsite
without further touch-ups.

Gate ACL waits on actual filter visibility: snapshotClientFilters +
waitForClientFilterChange ensure the new PacketFilter has reached the
client before assertions fire; SyncOption + WithPreBarrier feeds a
server-side policy-loaded check into WaitForTailscaleSyncPerUser.

Move advertise-routes mutation out of EventuallyWithT in route_test
(cmd/hi/README forbids retrying state-mutating calls). Pace the
TestNodeOnlineStatus outer loop with a Ticker, not a Sleep.
This commit is contained in:
Kristoffer Dalby 2026-05-13 13:19:49 +00:00
parent 78fd6efb38
commit dfcc96d808
6 changed files with 321 additions and 300 deletions

View file

@ -325,7 +325,7 @@ func TestACLHostsInNetMapTable(t *testing.T) {
user := status.User[status.Self.UserID].LoginName
assert.Len(c, status.Peer, (testCase.want[user]))
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "Waiting for expected peer visibility")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected peer visibility")
}
})
}
@ -372,10 +372,8 @@ func TestACLAllowUser80Dst(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user1 can reach user2")
assertCurlDockerHostname(c, client, url, "Verifying user1 can reach user2")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user1 can reach user2")
}
}
@ -390,7 +388,7 @@ func TestACLAllowUser80Dst(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, client, url, "user2 should not reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user2 cannot reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user2 cannot reach user1")
}
}
}
@ -437,7 +435,7 @@ func TestACLDenyAllPort80(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, client, url, "all traffic should be denied")
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying all traffic is denied")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying all traffic is denied")
}
}
}
@ -481,10 +479,8 @@ func TestACLAllowUserDst(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user1 can reach user2")
assertCurlDockerHostname(c, client, url, "Verifying user1 can reach user2")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user1 can reach user2")
}
}
@ -499,7 +495,7 @@ func TestACLAllowUserDst(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, client, url, "user2 should not reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user2 cannot reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user2 cannot reach user1")
}
}
}
@ -542,10 +538,8 @@ func TestACLAllowStarDst(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user1 can reach user2")
assertCurlDockerHostname(c, client, url, "Verifying user1 can reach user2")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user1 can reach user2")
}
}
@ -560,7 +554,7 @@ func TestACLAllowStarDst(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, client, url, "user2 should not reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user2 cannot reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user2 cannot reach user1")
}
}
}
@ -608,10 +602,8 @@ func TestACLNamedHostsCanReachBySubnet(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user1 can reach user2")
assertCurlDockerHostname(c, client, url, "Verifying user1 can reach user2")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user1 can reach user2")
}
}
@ -627,10 +619,8 @@ func TestACLNamedHostsCanReachBySubnet(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user2 can reach user1")
assertCurlDockerHostname(c, client, url, "Verifying user2 can reach user1")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user2 can reach user1")
}
}
}
@ -790,7 +780,7 @@ func TestACLNamedHostsCanReach(t *testing.T) {
test3URL,
result,
)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "test1 should reach test3")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "test1 should reach test3")
// test2 can query test3 (everyone -> test3)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
@ -804,7 +794,7 @@ func TestACLNamedHostsCanReach(t *testing.T) {
test3URL,
result,
)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "test2 should reach test3")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "test2 should reach test3")
// test3 cannot query test1
_, err = test3.CurlFailFast(test1URL)
@ -826,7 +816,7 @@ func TestACLNamedHostsCanReach(t *testing.T) {
test2URL,
result,
)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "test1 should reach test2")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "test1 should reach test2")
// test2 cannot query test1
_, err = test2.CurlFailFast(test1URL)
@ -981,12 +971,12 @@ func TestACLDevice1CanAccessDevice2(t *testing.T) {
test2URL,
result,
)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "test1 should reach test2")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "test1 should reach test2")
// test2 cannot query test1 (asymmetric policy)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, test2, test1URL, "test2 should not reach test1")
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "test2 should NOT reach test1")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "test2 should NOT reach test1")
})
}
}
@ -1047,10 +1037,8 @@ func TestPolicyUpdateWhileRunningWithCLIInDatabase(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying user1 can reach user2")
assertCurlDockerHostname(c, client, url, "Verifying user1 can reach user2")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying user1 can reach user2")
}
}
@ -1096,7 +1084,7 @@ func TestPolicyUpdateWhileRunningWithCLIInDatabase(t *testing.T) {
if diff := cmp.Diff(p, *output, cmpopts.IgnoreUnexported(policyv2.Policy{}), cmpopts.EquateEmpty()); diff != "" {
ct.Errorf("unexpected policy(-want +got):\n%s", diff)
}
}, integrationutil.ScaledTimeout(30*time.Second), 1*time.Second, "verifying that the new policy took place")
}, integrationutil.StatusReadyTimeout, 1*time.Second, "verifying that the new policy took place")
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
// Test that user1 can visit all user2
@ -1108,9 +1096,7 @@ func TestPolicyUpdateWhileRunningWithCLIInDatabase(t *testing.T) {
url := fmt.Sprintf("http://%s/etc/hostname", fqdn)
t.Logf("url from %s to %s", client.Hostname(), url)
result, err := client.Curl(url)
assert.Len(ct, result, 13)
assert.NoError(ct, err)
assertCurlDockerHostname(ct, client, url, "new policy did not get propagated to nodes")
}
}
@ -1126,7 +1112,7 @@ func TestPolicyUpdateWhileRunningWithCLIInDatabase(t *testing.T) {
assertCurlFailWithCollect(ct, client, url, "user2 should not reach user1")
}
}
}, integrationutil.ScaledTimeout(30*time.Second), 1*time.Second, "new policy did not get propagated to nodes")
}, integrationutil.StatusReadyTimeout, 1*time.Second, "new policy did not get propagated to nodes")
}
func TestACLAutogroupMember(t *testing.T) {
@ -1165,7 +1151,7 @@ func TestACLAutogroupMember(t *testing.T) {
clientIsUntagged = status.Self.Tags == nil || status.Self.Tags.Len() == 0
assert.True(c, clientIsUntagged, "Expected client %s to be untagged for autogroup:member test", client.Hostname())
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "Waiting for client %s to be untagged", client.Hostname())
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for client %s to be untagged", client.Hostname())
if !clientIsUntagged {
continue
@ -1184,7 +1170,7 @@ func TestACLAutogroupMember(t *testing.T) {
peerIsUntagged = status.Self.Tags == nil || status.Self.Tags.Len() == 0
assert.True(c, peerIsUntagged, "Expected peer %s to be untagged for autogroup:member test", peer.Hostname())
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "Waiting for peer %s to be untagged", peer.Hostname())
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for peer %s to be untagged", peer.Hostname())
if !peerIsUntagged {
continue
@ -1197,10 +1183,8 @@ func TestACLAutogroupMember(t *testing.T) {
t.Logf("url from %s to %s", client.Hostname(), url)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "Verifying autogroup:member connectivity")
assertCurlDockerHostname(c, client, url, "Verifying autogroup:member connectivity")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "Verifying autogroup:member connectivity")
}
}
}
@ -1376,7 +1360,7 @@ func TestACLAutogroupTagged(t *testing.T) {
untaggedClients = append(untaggedClients, client)
}
}
}, integrationutil.ScaledTimeout(30*time.Second), 1*time.Second, "verifying peer visibility for node %s", hostname)
}, integrationutil.StatusReadyTimeout, 1*time.Second, "verifying peer visibility for node %s", hostname)
}
// Verify we have the expected number of tagged and untagged nodes
@ -1390,7 +1374,7 @@ func TestACLAutogroupTagged(t *testing.T) {
assert.NoError(c, err)
assert.NotNil(c, status.Self.Tags, "tagged node %s should have tags", client.Hostname())
assert.Positive(c, status.Self.Tags.Len(), "tagged node %s should have at least one tag", client.Hostname())
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "Waiting for tags to be applied to tagged nodes")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for tags to be applied to tagged nodes")
}
// Verify untagged nodes have no tags
@ -1402,7 +1386,7 @@ func TestACLAutogroupTagged(t *testing.T) {
if status.Self.Tags != nil {
assert.Equal(c, 0, status.Self.Tags.Len(), "untagged node %s should have no tags", client.Hostname())
}
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "Waiting to verify untagged nodes have no tags")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting to verify untagged nodes have no tags")
}
// Test that tagged nodes can communicate with each other
@ -1420,10 +1404,8 @@ func TestACLAutogroupTagged(t *testing.T) {
t.Logf("Testing connection from tagged node %s to tagged node %s", client.Hostname(), peer.Hostname())
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(ct, err)
assert.Len(ct, result, 13)
}, integrationutil.ScaledTimeout(20*time.Second), 500*time.Millisecond, "tagged nodes should be able to communicate")
assertCurlDockerHostname(ct, client, url, "tagged nodes should be able to communicate")
}, integrationutil.ScaledTimeout(20*time.Second), integrationutil.SlowPoll, "tagged nodes should be able to communicate")
}
}
@ -1442,7 +1424,7 @@ func TestACLAutogroupTagged(t *testing.T) {
result, err := client.CurlFailFast(url)
assert.Empty(ct, result)
assert.Error(ct, err)
}, integrationutil.ScaledTimeout(5*time.Second), 200*time.Millisecond, "untagged nodes should not be able to reach tagged nodes")
}, integrationutil.ScaledTimeout(5*time.Second), integrationutil.FastPoll, "untagged nodes should not be able to reach tagged nodes")
}
// Try to reach other untagged nodes (should also fail)
@ -1462,7 +1444,7 @@ func TestACLAutogroupTagged(t *testing.T) {
result, err := client.CurlFailFast(url)
assert.Empty(ct, result)
assert.Error(ct, err)
}, integrationutil.ScaledTimeout(5*time.Second), 200*time.Millisecond, "untagged nodes should not be able to reach other untagged nodes")
}, integrationutil.ScaledTimeout(5*time.Second), integrationutil.FastPoll, "untagged nodes should not be able to reach other untagged nodes")
}
}
@ -1480,7 +1462,7 @@ func TestACLAutogroupTagged(t *testing.T) {
result, err := client.CurlFailFast(url)
assert.Empty(ct, result)
assert.Error(ct, err)
}, integrationutil.ScaledTimeout(5*time.Second), 200*time.Millisecond, "tagged nodes should not be able to reach untagged nodes")
}, integrationutil.ScaledTimeout(5*time.Second), integrationutil.FastPoll, "tagged nodes should not be able to reach untagged nodes")
}
}
}
@ -1664,10 +1646,8 @@ func TestACLAutogroupSelf(t *testing.T) {
t.Logf("url from %s (user1) to %s (user1)", client.Hostname(), fqdn)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "user1 device should reach other user1 device via autogroup:self")
assertCurlDockerHostname(c, client, url, "user1 device should reach other user1 device via autogroup:self")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "user1 device should reach other user1 device via autogroup:self")
}
}
@ -1685,10 +1665,8 @@ func TestACLAutogroupSelf(t *testing.T) {
t.Logf("url from %s (user2) to %s (user2)", client.Hostname(), fqdn)
assert.EventuallyWithT(t, func(c *assert.CollectT) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.Len(c, result, 13)
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "user2 device should reach other user2 device via autogroup:self")
assertCurlDockerHostname(c, client, url, "user2 device should reach other user2 device via autogroup:self")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "user2 device should reach other user2 device via autogroup:self")
}
}
@ -1704,7 +1682,7 @@ func TestACLAutogroupSelf(t *testing.T) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.NotEmpty(c, result, "user1 should be able to access router-node via group:home -> tag:router-node rule")
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "user1 device should reach router-node (proves autogroup:self doesn't interfere)")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "user1 device should reach router-node (proves autogroup:self doesn't interfere)")
}
// Test that user2's regular devices can access router-node
@ -1719,7 +1697,7 @@ func TestACLAutogroupSelf(t *testing.T) {
result, err := client.Curl(url)
assert.NoError(c, err)
assert.NotEmpty(c, result, "user2 should be able to access router-node via group:home -> tag:router-node rule")
}, integrationutil.ScaledTimeout(10*time.Second), 200*time.Millisecond, "user2 device should reach router-node (proves autogroup:self doesn't interfere)")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "user2 device should reach router-node (proves autogroup:self doesn't interfere)")
}
// Test that devices from different users cannot access each other's regular devices
@ -1867,20 +1845,32 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
assert.Len(ct, result, 13, "iteration %d: response from %s to %s should be valid", iteration, client.Hostname(), fqdn)
}
}
}, integrationutil.ScaledTimeout(90*time.Second), 500*time.Millisecond, "iteration %d: Phase 1 - all connectivity tests with allow-all policy", iteration)
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "iteration %d: Phase 1 - all connectivity tests with allow-all policy", iteration)
// Phase 2: Autogroup:self policy (only same user can access)
t.Logf("Iteration %d: Phase 2 - Setting autogroup:self policy", iteration)
preFilters := snapshotClientFilters(t, allClients)
err = headscale.SetPolicy(autogroupSelfPolicy)
require.NoError(t, err)
// Wait for peer lists to sync with autogroup:self - ensures cross-user peers are removed
t.Logf("Iteration %d: Phase 2 - Waiting for peer lists to sync with autogroup:self", iteration)
err = scenario.WaitForTailscaleSyncPerUser(integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond)
err = scenario.WaitForTailscaleSyncPerUser(
integrationutil.PolicyPropagationTimeout,
integrationutil.SlowPoll,
)
require.NoError(t, err, "iteration %d: Phase 2 - failed to sync after autogroup:self policy", iteration)
// Gate on each client's wgengine having applied the new
// PacketFilter before asserting reachability. WaitForTailscale
// SyncPerUser only checks peer-count parity, which trips
// before the filter rules have actually replaced the
// allow-all rules.
waitForClientFilterChange(t, allClients, preFilters, integrationutil.PolicyPropagationTimeout)
// Test ALL connectivity (positive and negative) in one block after state is settled
t.Logf("Iteration %d: Phase 2 - Testing all connectivity with autogroup:self", iteration)
assert.EventuallyWithT(t, func(ct *assert.CollectT) {
@ -1947,7 +1937,7 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
assertCurlFailWithCollect(ct, client, url, fmt.Sprintf("iteration %d: user2 %s should NOT reach user1 %s", iteration, client.Hostname(), peer.Hostname()))
}
}
}, integrationutil.ScaledTimeout(90*time.Second), 500*time.Millisecond, "iteration %d: Phase 2 - all connectivity tests with autogroup:self", iteration)
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "iteration %d: Phase 2 - all connectivity tests with autogroup:self", iteration)
// Phase 2b: Add a new node to user1 and validate policy propagation
t.Logf("Iteration %d: Phase 2b - Adding new node to user1 during autogroup:self policy", iteration)
@ -2011,7 +2001,7 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
assertCurlFailWithCollect(ct, client, url, fmt.Sprintf("iteration %d: user1 %s should NOT reach user2 %s", iteration, client.Hostname(), peer.Hostname()))
}
}
}, integrationutil.ScaledTimeout(90*time.Second), 500*time.Millisecond, "iteration %d: Phase 2b - all connectivity tests after new node addition", iteration)
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "iteration %d: Phase 2b - all connectivity tests after new node addition", iteration)
// Delete the newly added node before Phase 3
t.Logf("Iteration %d: Phase 2b - Deleting the newly added node from user1", iteration)
@ -2033,7 +2023,7 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
nodeToDeleteID = node.GetId()
}
}
}, integrationutil.ScaledTimeout(10*time.Second), 500*time.Millisecond, "iteration %d: Phase 2b - listing nodes before deletion", iteration)
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.SlowPoll, "iteration %d: Phase 2b - listing nodes before deletion", iteration)
// Delete the node via headscale helper
t.Logf("Iteration %d: Phase 2b - Deleting node ID %d from headscale", iteration, nodeToDeleteID)
@ -2066,7 +2056,7 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
nodeListAfter, err := headscale.ListNodes("user1")
assert.NoError(ct, err, "failed to list nodes after deletion")
assert.Len(ct, nodeListAfter, 2, "iteration %d: should have 2 user1 nodes after deletion, got %d", iteration, len(nodeListAfter))
}, integrationutil.ScaledTimeout(10*time.Second), 500*time.Millisecond, "iteration %d: Phase 2b - node should be deleted", iteration)
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.SlowPoll, "iteration %d: Phase 2b - node should be deleted", iteration)
// Wait for sync after deletion to ensure peer counts are correct
// Use WaitForTailscaleSyncPerUser because autogroup:self is still active,
@ -2128,7 +2118,7 @@ func TestACLPolicyPropagationOverTime(t *testing.T) {
assertCurlFailWithCollect(ct, client, url, fmt.Sprintf("iteration %d: user2 %s should NOT reach user1 %s", iteration, client.Hostname(), peer.Hostname()))
}
}
}, integrationutil.ScaledTimeout(90*time.Second), 500*time.Millisecond, "iteration %d: Phase 3 - all connectivity tests with directional policy", iteration)
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "iteration %d: Phase 3 - all connectivity tests with directional policy", iteration)
t.Logf("=== Iteration %d/5 completed successfully - All 3 phases passed ===", iteration)
}
@ -2591,7 +2581,7 @@ func TestACLTagPropagation(t *testing.T) {
} else {
assertCurlFailWithCollect(c, sourceClient, targetURL, "initial access should fail")
}
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "verifying initial access state")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "verifying initial access state")
// Step 1b: Verify initial NetMap visibility
t.Logf("Step 1b: Verifying initial NetMap visibility (expect visible=%v)", tt.initialAccess)
@ -2614,7 +2604,7 @@ func TestACLTagPropagation(t *testing.T) {
} else {
assert.False(c, found, "Target should NOT be visible in NetMap initially")
}
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "verifying initial NetMap visibility")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "verifying initial NetMap visibility")
// Step 2: Apply tag change
t.Logf("Step 2: Setting tags on node %d to %v", targetNodeID, tt.tagChange)
@ -2632,7 +2622,7 @@ func TestACLTagPropagation(t *testing.T) {
if node != nil {
assert.ElementsMatch(c, tt.tagChange, node.GetTags(), "Tags should be updated")
}
}, integrationutil.ScaledTimeout(10*time.Second), 500*time.Millisecond, "verifying tag change applied")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.SlowPoll, "verifying tag change applied")
// Step 3: Verify final NetMap visibility first (fast signal that
// the MapResponse propagated to the client).
@ -2660,7 +2650,7 @@ func TestACLTagPropagation(t *testing.T) {
} else {
assert.False(c, found, "Target should NOT be visible in NetMap after tag change")
}
}, integrationutil.ScaledTimeout(120*time.Second), 500*time.Millisecond, "verifying NetMap visibility propagated after tag change")
}, integrationutil.HASlowConvergeTimeout, integrationutil.SlowPoll, "verifying NetMap visibility propagated after tag change")
// Step 4: Verify final access state (this is the key test for #2389).
// Even though Step 3 confirmed the MapResponse arrived, the full
@ -2674,7 +2664,7 @@ func TestACLTagPropagation(t *testing.T) {
} else {
assertCurlFailWithCollect(c, sourceClient, targetURL, "final access should fail after tag change")
}
}, integrationutil.ScaledTimeout(120*time.Second), 500*time.Millisecond, "verifying access propagated after tag change")
}, integrationutil.HASlowConvergeTimeout, integrationutil.SlowPoll, "verifying access propagated after tag change")
t.Logf("Test %s PASSED: Tag change propagated correctly", tt.name)
})
@ -2823,7 +2813,7 @@ func TestACLTagPropagationPortSpecific(t *testing.T) {
t.Log("Step 1: Verifying HTTP access with tag:webserver (should succeed)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlSuccessWithCollect(c, user2Node, targetURL, "HTTP should work with tag:webserver")
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "initial HTTP access with tag:webserver")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "initial HTTP access with tag:webserver")
// Step 2: Change tag from webserver to sshonly
t.Logf("Step 2: Changing tag from webserver to sshonly on node %d", targetNodeID)
@ -2844,7 +2834,7 @@ func TestACLTagPropagationPortSpecific(t *testing.T) {
if node != nil {
assert.ElementsMatch(c, []string{"tag:sshonly"}, node.GetTags(), "Tags should be updated to sshonly")
}
}, integrationutil.ScaledTimeout(10*time.Second), 500*time.Millisecond, "verifying tag change applied")
}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.SlowPoll, "verifying tag change applied")
// Step 3: Verify peer is still visible in NetMap (partial access, not full removal)
t.Log("Step 3: Verifying peer remains visible in NetMap after tag change")
@ -2863,7 +2853,7 @@ func TestACLTagPropagationPortSpecific(t *testing.T) {
}
assert.True(c, found, "Peer should still be visible with tag:sshonly (port 22 access)")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "peer visibility after tag change")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "peer visibility after tag change")
// Step 4: Verify HTTP on port 80 now fails (tag:sshonly only allows port 22).
// Port-specific filter changes are harder than peer removal because
@ -2872,7 +2862,7 @@ func TestACLTagPropagationPortSpecific(t *testing.T) {
t.Log("Step 4: Verifying HTTP access is now blocked (tag:sshonly only allows port 22)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
assertCurlFailWithCollect(c, user2Node, targetURL, "HTTP should fail with tag:sshonly (only port 22 allowed)")
}, integrationutil.ScaledTimeout(90*time.Second), 500*time.Millisecond, "HTTP blocked after tag change to sshonly")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "HTTP blocked after tag change to sshonly")
t.Log("Test PASSED: Port-specific ACL changes propagated correctly")
}
@ -2963,19 +2953,15 @@ func TestACLGroupWithUnknownUser(t *testing.T) {
t.Log("Testing connectivity: user1 -> user2 (should succeed despite unknown user in group)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should be able to reach user2")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "user1 should reach user2")
assertCurlDockerHostname(c, user1, url, "user1 should be able to reach user2")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "user1 should reach user2")
// Test that user2 can reach user1 (bidirectional)
t.Log("Testing connectivity: user2 -> user1 (should succeed despite unknown user in group)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should be able to reach user1")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "user2 should reach user1")
assertCurlDockerHostname(c, user2, url, "user2 should be able to reach user1")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "user2 should reach user1")
t.Log("Test PASSED: Valid users can communicate despite unknown user reference in group")
}
@ -3069,17 +3055,13 @@ func TestACLGroupAfterUserDeletion(t *testing.T) {
t.Log("Step 1: Verifying initial connectivity between all users")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should be able to reach user2 initially")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "initial user1 -> user2 connectivity")
assertCurlDockerHostname(c, user1, url, "user1 should be able to reach user2 initially")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "initial user1 -> user2 connectivity")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should be able to reach user1 initially")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(30*time.Second), 500*time.Millisecond, "initial user2 -> user1 connectivity")
assertCurlDockerHostname(c, user2, url, "user2 should be able to reach user1 initially")
}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "initial user2 -> user1 connectivity")
// Step 2: Get user3's node and user, then delete them
t.Log("Step 2: Deleting user3's node and user from headscale")
@ -3114,10 +3096,8 @@ func TestACLGroupAfterUserDeletion(t *testing.T) {
// Test that user1 can still reach user2
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should still be able to reach user2 after user3 deletion (stale cache)")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user2 after user3 deletion")
assertCurlDockerHostname(c, user1, url, "user1 should still be able to reach user2 after user3 deletion (stale cache)")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "user1 -> user2 after user3 deletion")
// Step 4: Create a NEW user - this triggers updatePolicyManagerUsers() which
// re-evaluates the policy. According to issue #2967, this is when the bug manifests:
@ -3139,18 +3119,14 @@ func TestACLGroupAfterUserDeletion(t *testing.T) {
// Test that user1 can still reach user2 AFTER the policy refresh triggered by user creation
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should still reach user2 after policy refresh (BUG if this fails)")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user2 after policy refresh (issue #2967)")
assertCurlDockerHostname(c, user1, url, "user1 should still reach user2 after policy refresh (BUG if this fails)")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user1 -> user2 after policy refresh (issue #2967)")
// Test that user2 can still reach user1
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should still reach user1 after policy refresh (BUG if this fails)")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user2 -> user1 after policy refresh (issue #2967)")
assertCurlDockerHostname(c, user2, url, "user2 should still reach user1 after policy refresh (BUG if this fails)")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user2 -> user1 after policy refresh (issue #2967)")
t.Log("Test PASSED: Remaining users can communicate after deleted user and policy refresh")
}
@ -3256,17 +3232,13 @@ func TestACLGroupDeletionExactReproduction(t *testing.T) {
t.Log("Step 1: Verifying initial connectivity (user1 <-> user3)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user3FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should reach user3")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user3")
assertCurlDockerHostname(c, user1, url, "user1 should reach user3")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user1 -> user3")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user3.Curl(url)
assert.NoError(c, err, "user3 should reach user1")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user3 -> user1")
assertCurlDockerHostname(c, user3, url, "user3 should reach user1")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user3 -> user1")
t.Log("Step 1: PASSED - initial connectivity works")
@ -3293,17 +3265,13 @@ func TestACLGroupDeletionExactReproduction(t *testing.T) {
t.Log("Step 3: Verifying connectivity STILL works after user2 deletion")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user3FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should still reach user3 after user2 deletion")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user3 after user2 deletion")
assertCurlDockerHostname(c, user1, url, "user1 should still reach user3 after user2 deletion")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user1 -> user3 after user2 deletion")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user3.Curl(url)
assert.NoError(c, err, "user3 should still reach user1 after user2 deletion")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user3 -> user1 after user2 deletion")
assertCurlDockerHostname(c, user3, url, "user3 should still reach user1 after user2 deletion")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user3 -> user1 after user2 deletion")
t.Log("Step 3: PASSED - connectivity works after user2 deletion")
@ -3322,17 +3290,13 @@ func TestACLGroupDeletionExactReproduction(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user3FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "BUG #2967: user1 should still reach user3 after user4 creation")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user3 after user4 creation (issue #2967)")
assertCurlDockerHostname(c, user1, url, "BUG #2967: user1 should still reach user3 after user4 creation")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user1 -> user3 after user4 creation (issue #2967)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user3.Curl(url)
assert.NoError(c, err, "BUG #2967: user3 should still reach user1 after user4 creation")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user3 -> user1 after user4 creation (issue #2967)")
assertCurlDockerHostname(c, user3, url, "BUG #2967: user3 should still reach user1 after user4 creation")
}, integrationutil.PolicyPropagationTimeout, integrationutil.SlowPoll, "user3 -> user1 after user4 creation (issue #2967)")
// Additional verification: check filter rules are not empty
filter, err := headscale.DebugFilter()
@ -3432,17 +3396,13 @@ func TestACLDynamicUnknownUserAddition(t *testing.T) {
t.Log("Step 1: Verifying initial connectivity with valid policy (no unknown users)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should reach user2")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "initial user1 -> user2")
assertCurlDockerHostname(c, user1, url, "user1 should reach user2")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "initial user1 -> user2")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should reach user1")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "initial user2 -> user1")
assertCurlDockerHostname(c, user2, url, "user2 should reach user1")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "initial user2 -> user1")
t.Log("Step 1: PASSED - connectivity works with valid policy")
@ -3483,17 +3443,13 @@ func TestACLDynamicUnknownUserAddition(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should STILL reach user2 after adding unknown user")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user2 after unknown user added")
assertCurlDockerHostname(c, user1, url, "user1 should STILL reach user2 after adding unknown user")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "user1 -> user2 after unknown user added")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should STILL reach user1 after adding unknown user")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user2 -> user1 after unknown user added")
assertCurlDockerHostname(c, user2, url, "user2 should STILL reach user1 after adding unknown user")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "user2 -> user1 after unknown user added")
t.Log("Step 3: PASSED - connectivity maintained after adding unknown user")
t.Log("Test PASSED: v0.28.0-beta.1 scenario - unknown user added dynamically, valid users still work")
@ -3589,17 +3545,13 @@ func TestACLDynamicUnknownUserRemoval(t *testing.T) {
t.Log("Step 1: Verifying connectivity with unknown user in policy (v2 graceful handling)")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should reach user2 even with unknown user in policy")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "initial user1 -> user2 with unknown")
assertCurlDockerHostname(c, user1, url, "user1 should reach user2 even with unknown user in policy")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "initial user1 -> user2 with unknown")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should reach user1 even with unknown user in policy")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "initial user2 -> user1 with unknown")
assertCurlDockerHostname(c, user2, url, "user2 should reach user1 even with unknown user in policy")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "initial user2 -> user1 with unknown")
t.Log("Step 1: PASSED - connectivity works even with unknown user (v2 graceful handling)")
@ -3638,17 +3590,13 @@ func TestACLDynamicUnknownUserRemoval(t *testing.T) {
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user2FQDN)
result, err := user1.Curl(url)
assert.NoError(c, err, "user1 should reach user2 after removing unknown user")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user1 -> user2 after unknown removed")
assertCurlDockerHostname(c, user1, url, "user1 should reach user2 after removing unknown user")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "user1 -> user2 after unknown removed")
assert.EventuallyWithT(t, func(c *assert.CollectT) {
url := fmt.Sprintf("http://%s/etc/hostname", user1FQDN)
result, err := user2.Curl(url)
assert.NoError(c, err, "user2 should reach user1 after removing unknown user")
assert.Len(c, result, 13, "expected hostname response")
}, integrationutil.ScaledTimeout(60*time.Second), 500*time.Millisecond, "user2 -> user1 after unknown removed")
assertCurlDockerHostname(c, user2, url, "user2 should reach user1 after removing unknown user")
}, integrationutil.HAConvergeTimeout, integrationutil.SlowPoll, "user2 -> user1 after unknown removed")
t.Log("Step 3: PASSED - connectivity maintained after removing unknown user")
t.Log("Test PASSED: Removing unknown users from policy works correctly")