Merge remote-tracking branch 'origin/main'

integration/README.md

@@ -1,25 +1,336 @@
 # Integration testing
 
-Headscale relies on integration testing to ensure we remain compatible with Tailscale.
+Headscale's integration tests start a real Headscale server and run
+scenarios against real Tailscale clients across supported versions, all
+inside Docker. They are the safety net that keeps us honest about
+Tailscale protocol compatibility.
 
-This is typically performed by starting a Headscale server and running a test "scenario"
-with an array of Tailscale clients and versions.
+This file documents **how to write** integration tests. For **how to
+run** them, see [`../cmd/hi/README.md`](../cmd/hi/README.md).
 
-Headscale's test framework and the current set of scenarios are defined in this directory.
+Tests live in files ending with `_test.go`; the framework lives in the
+rest of this directory (`scenario.go`, `tailscale.go`, helpers, and the
+`hsic/`, `tsic/`, `dockertestutil/` packages).
 
-Tests are located in files ending with `_test.go` and the framework are located in the rest.
+## Running tests
 
-## Running integration tests locally
-
-The easiest way to run tests locally is to use [act](https://github.com/nektos/act), a local GitHub Actions runner:
+For local runs, use [`cmd/hi`](../cmd/hi):
 
+```bash
+go run ./cmd/hi doctor
+go run ./cmd/hi run "TestPingAllByIP"
 ```
+
+Alternatively, [`act`](https://github.com/nektos/act) runs the GitHub
+Actions workflow locally:
+
+```bash
 act pull_request -W .github/workflows/test-integration.yaml
 ```
 
-Alternatively, the `docker run` command in each GitHub workflow file can be used.
+Each test runs as a separate workflow on GitHub Actions. To add a new
+test, run `go generate` inside `../cmd/gh-action-integration-generator/`
+and commit the generated workflow file.
 
-## Running integration tests on GitHub Actions
+## Framework overview
 
-Each test currently runs as a separate workflows in GitHub actions, to add new test, run
-`go generate` inside `../cmd/gh-action-integration-generator/` and commit the result.
+The integration framework has four layers:
+
+- **`scenario.go`** — `Scenario` orchestrates a test environment: a
+  Headscale server, one or more users, and a collection of Tailscale
+  clients. `NewScenario(spec)` returns a ready-to-use environment.
+- **`hsic/`** — "Headscale Integration Container": wraps a Headscale
+  server in Docker. Options for config, DB backend, DERP, OIDC, etc.
+- **`tsic/`** — "Tailscale Integration Container": wraps a single
+  Tailscale client. Options for version, hostname, auth method, etc.
+- **`dockertestutil/`** — low-level Docker helpers (networks, container
+  lifecycle, `IsRunningInContainer()` detection).
+
+Tests compose these pieces via `ScenarioSpec` and `CreateHeadscaleEnv`
+rather than calling Docker directly.
+
+## Required scaffolding
+
+### `IntegrationSkip(t)`
+
+**Every** integration test function must call `IntegrationSkip(t)` as
+its first statement. Without it, the test runs in the wrong environment
+and fails with confusing errors.
+
+```go
+func TestMyScenario(t *testing.T) {
+    IntegrationSkip(t)
+    // ... rest of the test
+}
+```
+
+`IntegrationSkip` is defined in `integration/scenario_test.go:15` and:
+
+- skips the test when not running inside the Docker test container
+  (`dockertestutil.IsRunningInContainer()`),
+- skips when `-short` is passed to `go test`.
+
+### Scenario setup
+
+The canonical setup creates users, clients, and the Headscale server in
+one shot:
+
+```go
+func TestMyScenario(t *testing.T) {
+    IntegrationSkip(t)
+    t.Parallel()
+
+    spec := ScenarioSpec{
+        NodesPerUser: 2,
+        Users:        []string{"alice", "bob"},
+    }
+    scenario, err := NewScenario(spec)
+    require.NoError(t, err)
+    defer scenario.ShutdownAssertNoPanics(t)
+
+    err = scenario.CreateHeadscaleEnv(
+        []tsic.Option{tsic.WithSSH()},
+        hsic.WithTestName("myscenario"),
+    )
+    require.NoError(t, err)
+
+    allClients, err := scenario.ListTailscaleClients()
+    require.NoError(t, err)
+
+    headscale, err := scenario.Headscale()
+    require.NoError(t, err)
+
+    // ... assertions
+}
+```
+
+Review `scenario.go` and `hsic/options.go` / `tsic/options.go` for the
+full option set (DERP, OIDC, policy files, DB backend, ACL grants,
+exit-node config, etc.).
+
+## The `EventuallyWithT` pattern
+
+Integration tests operate on a distributed system with real async
+propagation: clients advertise state, the server processes it, updates
+stream to peers. Direct assertions after state changes fail
+intermittently. Wrap external calls in `assert.EventuallyWithT`:
+
+```go
+assert.EventuallyWithT(t, func(c *assert.CollectT) {
+    status, err := client.Status()
+    assert.NoError(c, err)
+    for _, peerKey := range status.Peers() {
+        peerStatus := status.Peer[peerKey]
+        requirePeerSubnetRoutesWithCollect(c, peerStatus, expectedRoutes)
+    }
+}, 10*time.Second, 500*time.Millisecond, "client should see expected routes")
+```
+
+### External calls that need wrapping
+
+These read distributed state and may reflect stale data until
+propagation completes:
+
+- `headscale.ListNodes()`
+- `client.Status()`
+- `client.Curl()`
+- `client.Traceroute()`
+- `client.Execute()` when the command reads state
+
+### Blocking operations that must NOT be wrapped
+
+State-mutating commands run exactly once and either succeed or fail
+immediately — not eventually. Wrapping them in `EventuallyWithT` hides
+real failures behind retry.
+
+Use `client.MustStatus()` when you only need an ID for a blocking call:
+
+```go
+// CORRECT — mutation runs once
+for _, client := range allClients {
+    status := client.MustStatus()
+    _, _, err := client.Execute([]string{
+        "tailscale", "set",
+        "--advertise-routes=" + expectedRoutes[string(status.Self.ID)],
+    })
+    require.NoErrorf(t, err, "failed to advertise route: %s", err)
+}
+```
+
+Typical blocking operations: any `tailscale set` (routes, exit node,
+accept-routes, ssh), node registration via the CLI, user creation via
+gRPC.
+
+### The four rules
+
+1. **One external call per `EventuallyWithT` block.** Related assertions
+   on the result of a single call go together in the same block.
+
+   **Loop exception**: iterating over a collection of clients (or peers)
+   and calling `Status()` on each inside a single block is allowed — it
+   is the same logical "check all clients" operation. The rule applies
+   to distinct calls like `ListNodes()` + `Status()`, which must be
+   split into separate blocks.
+
+2. **Never nest `EventuallyWithT` calls.** A nested retry loop
+   multiplies timing windows and makes failures impossible to diagnose.
+
+3. **Use `*WithCollect` helper variants** inside the block. Regular
+   helpers use `require` and abort on the first failed assertion,
+   preventing retry.
+
+4. **Always provide a descriptive final message** — it appears on
+   failure and is your only clue about what the test was waiting for.
+
+### Variable scoping
+
+Variables used across multiple `EventuallyWithT` blocks must be declared
+at function scope. Inside the block, assign with `=`, not `:=` — `:=`
+creates a shadow invisible to the outer scope:
+
+```go
+var nodes []*v1.Node
+var err error
+assert.EventuallyWithT(t, func(c *assert.CollectT) {
+    nodes, err = headscale.ListNodes()   // = not :=
+    assert.NoError(c, err)
+    assert.Len(c, nodes, 2)
+    requireNodeRouteCountWithCollect(c, nodes[0], 2, 2, 2)
+}, 10*time.Second, 500*time.Millisecond, "nodes should have expected routes")
+
+// nodes is usable here because it was declared at function scope
+```
+
+### Helper functions
+
+Inside `EventuallyWithT` blocks, use the `*WithCollect` variants so
+assertion failures restart the wait loop instead of failing the test
+immediately:
+
+- `requirePeerSubnetRoutesWithCollect(c, status, expected)` —
+  `integration/route_test.go:2941`
+- `requireNodeRouteCountWithCollect(c, node, announced, approved, subnet)` —
+  `integration/route_test.go:2958`
+- `assertTracerouteViaIPWithCollect(c, traceroute, ip)` —
+  `integration/route_test.go:2898`
+
+When you write a new helper to be called inside `EventuallyWithT`, it
+must accept `*assert.CollectT` as its first parameter, not `*testing.T`.
+
+## Identifying nodes by property, not position
+
+The order of `headscale.ListNodes()` is not stable. Tests that index
+`nodes[0]` will break when node ordering changes. Look nodes up by ID,
+hostname, or tag:
+
+```go
+// WRONG — relies on array position
+require.Len(t, nodes[0].GetAvailableRoutes(), 1)
+
+// CORRECT — find the node that should have the route
+expectedRoutes := map[string]string{"1": "10.33.0.0/16"}
+for _, node := range nodes {
+    nodeIDStr := fmt.Sprintf("%d", node.GetId())
+    if route, shouldHaveRoute := expectedRoutes[nodeIDStr]; shouldHaveRoute {
+        assert.Contains(t, node.GetAvailableRoutes(), route)
+    }
+}
+```
+
+## Full example: advertising and approving a route
+
+```go
+func TestRouteAdvertisementBasic(t *testing.T) {
+    IntegrationSkip(t)
+    t.Parallel()
+
+    spec := ScenarioSpec{
+        NodesPerUser: 2,
+        Users:        []string{"user1"},
+    }
+    scenario, err := NewScenario(spec)
+    require.NoError(t, err)
+    defer scenario.ShutdownAssertNoPanics(t)
+
+    err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("route"))
+    require.NoError(t, err)
+
+    allClients, err := scenario.ListTailscaleClients()
+    require.NoError(t, err)
+
+    headscale, err := scenario.Headscale()
+    require.NoError(t, err)
+
+    // --- Blocking: advertise the route on one client ---
+    router := allClients[0]
+    _, _, err = router.Execute([]string{
+        "tailscale", "set",
+        "--advertise-routes=10.33.0.0/16",
+    })
+    require.NoErrorf(t, err, "advertising route: %s", err)
+
+    // --- Eventually: headscale should see the announced route ---
+    var nodes []*v1.Node
+    assert.EventuallyWithT(t, func(c *assert.CollectT) {
+        nodes, err = headscale.ListNodes()
+        assert.NoError(c, err)
+        assert.Len(c, nodes, 2)
+
+        for _, node := range nodes {
+            if node.GetName() == router.Hostname() {
+                requireNodeRouteCountWithCollect(c, node, 1, 0, 0)
+            }
+        }
+    }, 10*time.Second, 500*time.Millisecond, "route should be announced")
+
+    // --- Blocking: approve the route via headscale CLI ---
+    var routerNode *v1.Node
+    for _, node := range nodes {
+        if node.GetName() == router.Hostname() {
+            routerNode = node
+            break
+        }
+    }
+    require.NotNil(t, routerNode)
+
+    _, err = headscale.ApproveRoutes(routerNode.GetId(), []string{"10.33.0.0/16"})
+    require.NoError(t, err)
+
+    // --- Eventually: a peer should see the approved route ---
+    peer := allClients[1]
+    assert.EventuallyWithT(t, func(c *assert.CollectT) {
+        status, err := peer.Status()
+        assert.NoError(c, err)
+        for _, peerKey := range status.Peers() {
+            if peerKey == router.PublicKey() {
+                requirePeerSubnetRoutesWithCollect(c,
+                    status.Peer[peerKey],
+                    []netip.Prefix{netip.MustParsePrefix("10.33.0.0/16")})
+            }
+        }
+    }, 10*time.Second, 500*time.Millisecond, "peer should see approved route")
+}
+```
+
+## Common pitfalls
+
+- **Forgetting `IntegrationSkip(t)`**: the test runs outside Docker and
+  fails in confusing ways. Always the first line.
+- **Using `require` inside `EventuallyWithT`**: aborts after the first
+  iteration instead of retrying. Use `assert.*` + the `*WithCollect`
+  helpers.
+- **Mixing mutation and query in one `EventuallyWithT`**: hides real
+  failures. Keep mutation outside, query inside.
+- **Assuming node ordering**: look up by property.
+- **Ignoring `err` from `client.Status()`**: retry only retries the
+  whole block; don't silently drop errors from mid-block calls.
+- **Timeouts too tight**: 5s is reasonable for local state, 10s for
+  state that must propagate through the map poll cycle. Don't go lower
+  to "speed up the test" — you just make it flaky.
+
+## Debugging failing tests
+
+Tests save comprehensive artefacts to `control_logs/{runID}/`. Read them
+in this order: server stderr, client stderr, MapResponse JSON, database
+snapshot. The full debugging workflow, heuristics, and failure patterns
+are documented in [`../cmd/hi/README.md`](../cmd/hi/README.md).

integration/api_auth_test.go

@@ -0,0 +1,700 @@
+package integration
+
+import (
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/encoding/protojson"
+)
+
+// TestAPIAuthenticationBypass tests that the API authentication middleware
+// properly blocks unauthorized requests and does not leak sensitive data.
+// This test reproduces the security issue described in:
+// - https://github.com/juanfont/headscale/issues/2809
+// - https://github.com/juanfont/headscale/pull/2810
+//
+// The bug: When authentication fails, the middleware writes "Unauthorized"
+// but doesn't return early, allowing the handler to execute and append
+// sensitive data to the response.
+func TestAPIAuthenticationBypass(t *testing.T) {
+	IntegrationSkip(t)
+
+	spec := ScenarioSpec{
+		Users: []string{"user1", "user2", "user3"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("apiauthbypass"))
+	require.NoError(t, err)
+
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	// Create an API key using the CLI
+	var validAPIKey string
+
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		apiKeyOutput, err := headscale.Execute(
+			[]string{
+				"headscale",
+				"apikeys",
+				"create",
+				"--expiration",
+				"24h",
+			},
+		)
+		assert.NoError(ct, err)
+		assert.NotEmpty(ct, apiKeyOutput)
+		validAPIKey = strings.TrimSpace(apiKeyOutput)
+	}, integrationutil.ScaledTimeout(20*time.Second), 1*time.Second)
+
+	// Get the API endpoint
+	endpoint := headscale.GetEndpoint()
+	apiURL := endpoint + "/api/v1/user"
+
+	// Create HTTP client
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec
+		},
+	}
+
+	t.Run("HTTP_NoAuthHeader", func(t *testing.T) {
+		// Test 1: Request without any Authorization header
+		// Expected: Should return 401 with ONLY "Unauthorized" text, no user data
+		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil)
+		require.NoError(t, err)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		// Should return 401 Unauthorized
+		assert.Equal(t, http.StatusUnauthorized, resp.StatusCode,
+			"Expected 401 status code for request without auth header")
+
+		bodyStr := string(body)
+
+		// Should contain "Unauthorized" message
+		assert.Contains(t, bodyStr, "Unauthorized",
+			"Response should contain 'Unauthorized' message")
+
+		// Should NOT contain user data after "Unauthorized"
+		// This is the security bypass - if users array is present, auth was bypassed
+		var jsonCheck map[string]any
+
+		jsonErr := json.Unmarshal(body, &jsonCheck)
+
+		// If we can unmarshal JSON and it contains "users", that's the bypass
+		if jsonErr == nil {
+			assert.NotContains(t, jsonCheck, "users",
+				"SECURITY ISSUE: Response should NOT contain 'users' data when unauthorized")
+			assert.NotContains(t, jsonCheck, "user",
+				"SECURITY ISSUE: Response should NOT contain 'user' data when unauthorized")
+		}
+
+		// Additional check: response should not contain "user1", "user2", "user3"
+		assert.NotContains(t, bodyStr, "user1",
+			"SECURITY ISSUE: Response should NOT leak user 'user1' data")
+		assert.NotContains(t, bodyStr, "user2",
+			"SECURITY ISSUE: Response should NOT leak user 'user2' data")
+		assert.NotContains(t, bodyStr, "user3",
+			"SECURITY ISSUE: Response should NOT leak user 'user3' data")
+
+		// Response should be minimal, just "Unauthorized"
+		// Allow some variation in response format but body should be small
+		assert.Less(t, len(bodyStr), 100,
+			"SECURITY ISSUE: Unauthorized response body should be minimal, got: %s", bodyStr)
+	})
+
+	t.Run("HTTP_InvalidAuthHeader", func(t *testing.T) {
+		// Test 2: Request with invalid Authorization header (missing "Bearer " prefix)
+		// Expected: Should return 401 with ONLY "Unauthorized" text, no user data
+		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil)
+		require.NoError(t, err)
+		req.Header.Set("Authorization", "InvalidToken")
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusUnauthorized, resp.StatusCode,
+			"Expected 401 status code for invalid auth header format")
+
+		bodyStr := string(body)
+		assert.Contains(t, bodyStr, "Unauthorized")
+
+		// Should not leak user data
+		assert.NotContains(t, bodyStr, "user1",
+			"SECURITY ISSUE: Response should NOT leak user data")
+		assert.NotContains(t, bodyStr, "user2",
+			"SECURITY ISSUE: Response should NOT leak user data")
+		assert.NotContains(t, bodyStr, "user3",
+			"SECURITY ISSUE: Response should NOT leak user data")
+
+		assert.Less(t, len(bodyStr), 100,
+			"SECURITY ISSUE: Unauthorized response should be minimal")
+	})
+
+	t.Run("HTTP_InvalidBearerToken", func(t *testing.T) {
+		// Test 3: Request with Bearer prefix but invalid token
+		// Expected: Should return 401 with ONLY "Unauthorized" text, no user data
+		// Note: Both malformed and properly formatted invalid tokens should return 401
+		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil)
+		require.NoError(t, err)
+		req.Header.Set("Authorization", "Bearer invalid-token-12345")
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		assert.Equal(t, http.StatusUnauthorized, resp.StatusCode,
+			"Expected 401 status code for invalid bearer token")
+
+		bodyStr := string(body)
+		assert.Contains(t, bodyStr, "Unauthorized")
+
+		// Should not leak user data
+		assert.NotContains(t, bodyStr, "user1",
+			"SECURITY ISSUE: Response should NOT leak user data")
+		assert.NotContains(t, bodyStr, "user2",
+			"SECURITY ISSUE: Response should NOT leak user data")
+		assert.NotContains(t, bodyStr, "user3",
+			"SECURITY ISSUE: Response should NOT leak user data")
+
+		assert.Less(t, len(bodyStr), 100,
+			"SECURITY ISSUE: Unauthorized response should be minimal")
+	})
+
+	t.Run("HTTP_ValidAPIKey", func(t *testing.T) {
+		// Test 4: Request with valid API key
+		// Expected: Should return 200 with user data (this is the authorized case)
+		req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, apiURL, nil)
+		require.NoError(t, err)
+		req.Header.Set("Authorization", "Bearer "+validAPIKey)
+
+		resp, err := client.Do(req)
+		require.NoError(t, err)
+
+		defer resp.Body.Close()
+
+		body, err := io.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		// Should succeed with valid auth
+		assert.Equal(t, http.StatusOK, resp.StatusCode,
+			"Expected 200 status code with valid API key")
+
+		// Should be able to parse as protobuf JSON
+		var response v1.ListUsersResponse
+
+		err = protojson.Unmarshal(body, &response)
+		require.NoError(t, err, "Response should be valid protobuf JSON with valid API key")
+
+		// Should contain our test users
+		users := response.GetUsers()
+		assert.Len(t, users, 3, "Should have 3 users")
+
+		userNames := make([]string, len(users))
+		for i, u := range users {
+			userNames[i] = u.GetName()
+		}
+
+		assert.Contains(t, userNames, "user1")
+		assert.Contains(t, userNames, "user2")
+		assert.Contains(t, userNames, "user3")
+	})
+}
+
+// TestAPIAuthenticationBypassCurl tests the same security issue using curl
+// from inside a container, which is closer to how the issue was discovered.
+func TestAPIAuthenticationBypassCurl(t *testing.T) {
+	IntegrationSkip(t)
+
+	spec := ScenarioSpec{
+		Users: []string{"testuser1", "testuser2"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("apiauthcurl"))
+	require.NoError(t, err)
+
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	// Create a valid API key
+	apiKeyOutput, err := headscale.Execute(
+		[]string{
+			"headscale",
+			"apikeys",
+			"create",
+			"--expiration",
+			"24h",
+		},
+	)
+	require.NoError(t, err)
+
+	validAPIKey := strings.TrimSpace(apiKeyOutput)
+
+	endpoint := headscale.GetEndpoint()
+	apiURL := endpoint + "/api/v1/user"
+
+	t.Run("Curl_NoAuth", func(t *testing.T) {
+		// Execute curl from inside the headscale container without auth
+		curlOutput, err := headscale.Execute(
+			[]string{
+				"curl",
+				"-s",
+				"-w",
+				"\nHTTP_CODE:%{http_code}",
+				apiURL,
+			},
+		)
+		require.NoError(t, err)
+
+		// Parse the output
+		lines := strings.Split(curlOutput, "\n")
+
+		var (
+			httpCode     string
+			responseBody string
+		)
+
+		var responseBodySb280 strings.Builder
+
+		for _, line := range lines {
+			if after, ok := strings.CutPrefix(line, "HTTP_CODE:"); ok {
+				httpCode = after
+			} else {
+				responseBodySb280.WriteString(line)
+			}
+		}
+
+		responseBody += responseBodySb280.String()
+
+		// Should return 401
+		assert.Equal(t, "401", httpCode,
+			"Curl without auth should return 401")
+
+		// Should contain Unauthorized
+		assert.Contains(t, responseBody, "Unauthorized",
+			"Response should contain 'Unauthorized'")
+
+		// Should NOT leak user data
+		assert.NotContains(t, responseBody, "testuser1",
+			"SECURITY ISSUE: Should not leak user data")
+		assert.NotContains(t, responseBody, "testuser2",
+			"SECURITY ISSUE: Should not leak user data")
+
+		// Response should be small (just "Unauthorized")
+		assert.Less(t, len(responseBody), 100,
+			"SECURITY ISSUE: Unauthorized response should be minimal, got: %s", responseBody)
+	})
+
+	t.Run("Curl_InvalidAuth", func(t *testing.T) {
+		// Execute curl with invalid auth header
+		curlOutput, err := headscale.Execute(
+			[]string{
+				"curl",
+				"-s",
+				"-H",
+				"Authorization: InvalidToken",
+				"-w",
+				"\nHTTP_CODE:%{http_code}",
+				apiURL,
+			},
+		)
+		require.NoError(t, err)
+
+		lines := strings.Split(curlOutput, "\n")
+
+		var (
+			httpCode     string
+			responseBody string
+		)
+
+		var responseBodySb326 strings.Builder
+
+		for _, line := range lines {
+			if after, ok := strings.CutPrefix(line, "HTTP_CODE:"); ok {
+				httpCode = after
+			} else {
+				responseBodySb326.WriteString(line)
+			}
+		}
+
+		responseBody += responseBodySb326.String()
+
+		assert.Equal(t, "401", httpCode)
+		assert.Contains(t, responseBody, "Unauthorized")
+		assert.NotContains(t, responseBody, "testuser1",
+			"SECURITY ISSUE: Should not leak user data")
+		assert.NotContains(t, responseBody, "testuser2",
+			"SECURITY ISSUE: Should not leak user data")
+	})
+
+	t.Run("Curl_ValidAuth", func(t *testing.T) {
+		// Execute curl with valid API key
+		curlOutput, err := headscale.Execute(
+			[]string{
+				"curl",
+				"-s",
+				"-H",
+				"Authorization: Bearer " + validAPIKey,
+				"-w",
+				"\nHTTP_CODE:%{http_code}",
+				apiURL,
+			},
+		)
+		require.NoError(t, err)
+
+		lines := strings.Split(curlOutput, "\n")
+
+		var (
+			httpCode     string
+			responseBody string
+		)
+
+		var responseBodySb361 strings.Builder
+
+		for _, line := range lines {
+			if after, ok := strings.CutPrefix(line, "HTTP_CODE:"); ok {
+				httpCode = after
+			} else {
+				responseBodySb361.WriteString(line)
+			}
+		}
+
+		responseBody += responseBodySb361.String()
+
+		// Should succeed
+		assert.Equal(t, "200", httpCode,
+			"Curl with valid API key should return 200")
+
+		// Should contain user data
+		var response v1.ListUsersResponse
+
+		err = protojson.Unmarshal([]byte(responseBody), &response)
+		require.NoError(t, err, "Response should be valid protobuf JSON")
+
+		users := response.GetUsers()
+		assert.Len(t, users, 2, "Should have 2 users")
+	})
+}
+
+// TestGRPCAuthenticationBypass tests that the gRPC authentication interceptor
+// properly blocks unauthorized requests.
+// This test verifies that the gRPC API does not have the same bypass issue
+// as the HTTP API middleware.
+func TestGRPCAuthenticationBypass(t *testing.T) {
+	IntegrationSkip(t)
+
+	spec := ScenarioSpec{
+		Users: []string{"grpcuser1", "grpcuser2"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	// We need TLS for remote gRPC connections
+	err = scenario.CreateHeadscaleEnv(
+		[]tsic.Option{},
+		hsic.WithTestName("grpcauthtest"),
+		hsic.WithConfigEnv(map[string]string{
+			// Enable gRPC on the standard port
+			"HEADSCALE_GRPC_LISTEN_ADDR": "0.0.0.0:50443",
+		}),
+	)
+	require.NoError(t, err)
+
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	// Create a valid API key
+	apiKeyOutput, err := headscale.Execute(
+		[]string{
+			"headscale",
+			"apikeys",
+			"create",
+			"--expiration",
+			"24h",
+		},
+	)
+	require.NoError(t, err)
+
+	validAPIKey := strings.TrimSpace(apiKeyOutput)
+
+	// Get the gRPC endpoint
+	// For gRPC, we need to use the hostname and port 50443
+	grpcAddress := headscale.GetHostname() + ":50443"
+
+	t.Run("gRPC_NoAPIKey", func(t *testing.T) {
+		// Test 1: Try to use CLI without API key (should fail)
+		// When HEADSCALE_CLI_ADDRESS is set but HEADSCALE_CLI_API_KEY is not set,
+		// the CLI should fail immediately
+		_, err := headscale.Execute(
+			[]string{
+				"sh", "-c",
+				fmt.Sprintf("HEADSCALE_CLI_ADDRESS=%s HEADSCALE_CLI_INSECURE=true headscale users list --output json 2>&1", grpcAddress),
+			},
+		)
+
+		// Should fail - CLI exits when API key is missing
+		assert.Error(t, err,
+			"gRPC connection without API key should fail")
+	})
+
+	t.Run("gRPC_InvalidAPIKey", func(t *testing.T) {
+		// Test 2: Try to use CLI with invalid API key (should fail with auth error)
+		output, err := headscale.Execute(
+			[]string{
+				"sh", "-c",
+				fmt.Sprintf("HEADSCALE_CLI_ADDRESS=%s HEADSCALE_CLI_API_KEY=invalid-key-12345 HEADSCALE_CLI_INSECURE=true headscale users list --output json 2>&1", grpcAddress),
+			},
+		)
+
+		// Should fail with authentication error
+		require.Error(t, err,
+			"gRPC connection with invalid API key should fail")
+
+		// Should contain authentication error message
+		outputStr := strings.ToLower(output)
+		assert.True(t,
+			strings.Contains(outputStr, "unauthenticated") ||
+				strings.Contains(outputStr, "invalid token") ||
+				strings.Contains(outputStr, "validating token") ||
+				strings.Contains(outputStr, "authentication"),
+			"Error should indicate authentication failure, got: %s", output)
+
+		// Should NOT leak user data
+		assert.NotContains(t, output, "grpcuser1",
+			"SECURITY ISSUE: gRPC should not leak user data with invalid auth")
+		assert.NotContains(t, output, "grpcuser2",
+			"SECURITY ISSUE: gRPC should not leak user data with invalid auth")
+	})
+
+	t.Run("gRPC_ValidAPIKey", func(t *testing.T) {
+		// Test 3: Use CLI with valid API key (should succeed)
+		output, err := headscale.Execute(
+			[]string{
+				"sh", "-c",
+				fmt.Sprintf("HEADSCALE_CLI_ADDRESS=%s HEADSCALE_CLI_API_KEY=%s HEADSCALE_CLI_INSECURE=true headscale users list --output json", grpcAddress, validAPIKey),
+			},
+		)
+
+		// Should succeed
+		require.NoError(t, err,
+			"gRPC connection with valid API key should succeed, output: %s", output)
+
+		// CLI outputs the users array directly, not wrapped in [v1.ListUsersResponse]
+		// Parse as JSON array (CLI uses [json.Marshal], not protojson)
+		var users []*v1.User
+
+		err = json.Unmarshal([]byte(output), &users)
+		require.NoError(t, err, "Response should be valid JSON array")
+		assert.Len(t, users, 2, "Should have 2 users")
+
+		userNames := make([]string, len(users))
+		for i, u := range users {
+			userNames[i] = u.GetName()
+		}
+
+		assert.Contains(t, userNames, "grpcuser1")
+		assert.Contains(t, userNames, "grpcuser2")
+	})
+}
+
+// TestCLIWithConfigAuthenticationBypass tests that the headscale CLI
+// with --config flag does not have authentication bypass issues when
+// connecting to a remote server.
+// Note: When using --config with local unix socket, no auth is needed.
+// This test focuses on remote gRPC connections which require API keys.
+func TestCLIWithConfigAuthenticationBypass(t *testing.T) {
+	IntegrationSkip(t)
+
+	spec := ScenarioSpec{
+		Users: []string{"cliuser1", "cliuser2"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv(
+		[]tsic.Option{},
+		hsic.WithTestName("cliconfigauth"),
+		hsic.WithConfigEnv(map[string]string{
+			"HEADSCALE_GRPC_LISTEN_ADDR": "0.0.0.0:50443",
+		}),
+	)
+	require.NoError(t, err)
+
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	// Create a valid API key
+	apiKeyOutput, err := headscale.Execute(
+		[]string{
+			"headscale",
+			"apikeys",
+			"create",
+			"--expiration",
+			"24h",
+		},
+	)
+	require.NoError(t, err)
+
+	validAPIKey := strings.TrimSpace(apiKeyOutput)
+
+	grpcAddress := headscale.GetHostname() + ":50443"
+
+	// Create a config file for testing
+	configWithoutKey := fmt.Sprintf(`
+cli:
+  address: %s
+  timeout: 5s
+  insecure: true
+`, grpcAddress)
+
+	configWithInvalidKey := fmt.Sprintf(`
+cli:
+  address: %s
+  api_key: invalid-key-12345
+  timeout: 5s
+  insecure: true
+`, grpcAddress)
+
+	configWithValidKey := fmt.Sprintf(`
+cli:
+  address: %s
+  api_key: %s
+  timeout: 5s
+  insecure: true
+`, grpcAddress, validAPIKey)
+
+	t.Run("CLI_Config_NoAPIKey", func(t *testing.T) {
+		// Create config file without API key
+		err := headscale.WriteFile("/tmp/config_no_key.yaml", []byte(configWithoutKey))
+		require.NoError(t, err)
+
+		// Try to use CLI with config that has no API key
+		_, err = headscale.Execute(
+			[]string{
+				"headscale",
+				"--config", "/tmp/config_no_key.yaml",
+				"users", "list",
+				"--output", "json",
+			},
+		)
+
+		// Should fail
+		assert.Error(t, err,
+			"CLI with config missing API key should fail")
+	})
+
+	t.Run("CLI_Config_InvalidAPIKey", func(t *testing.T) {
+		// Create config file with invalid API key
+		err := headscale.WriteFile("/tmp/config_invalid_key.yaml", []byte(configWithInvalidKey))
+		require.NoError(t, err)
+
+		// Try to use CLI with invalid API key
+		output, err := headscale.Execute(
+			[]string{
+				"sh", "-c",
+				"headscale --config /tmp/config_invalid_key.yaml users list --output json 2>&1",
+			},
+		)
+
+		// Should fail
+		require.Error(t, err,
+			"CLI with invalid API key should fail")
+
+		// Should indicate authentication failure
+		outputStr := strings.ToLower(output)
+		assert.True(t,
+			strings.Contains(outputStr, "unauthenticated") ||
+				strings.Contains(outputStr, "invalid token") ||
+				strings.Contains(outputStr, "validating token") ||
+				strings.Contains(outputStr, "authentication"),
+			"Error should indicate authentication failure, got: %s", output)
+
+		// Should NOT leak user data
+		assert.NotContains(t, output, "cliuser1",
+			"SECURITY ISSUE: CLI should not leak user data with invalid auth")
+		assert.NotContains(t, output, "cliuser2",
+			"SECURITY ISSUE: CLI should not leak user data with invalid auth")
+	})
+
+	t.Run("CLI_Config_ValidAPIKey", func(t *testing.T) {
+		// Create config file with valid API key
+		err := headscale.WriteFile("/tmp/config_valid_key.yaml", []byte(configWithValidKey))
+		require.NoError(t, err)
+
+		// Use CLI with valid API key
+		output, err := headscale.Execute(
+			[]string{
+				"headscale",
+				"--config", "/tmp/config_valid_key.yaml",
+				"users", "list",
+				"--output", "json",
+			},
+		)
+
+		// Should succeed
+		require.NoError(t, err,
+			"CLI with valid API key should succeed")
+
+		// CLI outputs the users array directly, not wrapped in [v1.ListUsersResponse]
+		// Parse as JSON array (CLI uses [json.Marshal], not protojson)
+		var users []*v1.User
+
+		err = json.Unmarshal([]byte(output), &users)
+		require.NoError(t, err, "Response should be valid JSON array")
+		assert.Len(t, users, 2, "Should have 2 users")
+
+		userNames := make([]string, len(users))
+		for i, u := range users {
+			userNames[i] = u.GetName()
+		}
+
+		assert.Contains(t, userNames, "cliuser1")
+		assert.Contains(t, userNames, "cliuser2")
+	})
+}

integration/auth_key_test.go

@@ -9,12 +9,15 @@ import (
 	"time"
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
 	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
 )
 
 func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
@@ -28,66 +31,67 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 			}
 
 			scenario, err := NewScenario(spec)
-			assertNoErr(t, err)
+
+			require.NoError(t, err)
 			defer scenario.ShutdownAssertNoPanics(t)
 
 			opts := []hsic.Option{
-				hsic.WithTestName("pingallbyip"),
-				hsic.WithEmbeddedDERPServerOnly(),
-				hsic.WithDERPAsIP(),
-			}
-			if https {
-				opts = append(opts, []hsic.Option{
-					hsic.WithTLS(),
-				}...)
+				hsic.WithTestName("authkey-relogsame"),
 			}
 
 			err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
-			assertNoErrHeadscaleEnv(t, err)
+			requireNoErrHeadscaleEnv(t, err)
 
 			allClients, err := scenario.ListTailscaleClients()
-			assertNoErrListClients(t, err)
+			requireNoErrListClients(t, err)
 
 			allIps, err := scenario.ListTailscaleClientsIPs()
-			assertNoErrListClientIPs(t, err)
+			requireNoErrListClientIPs(t, err)
 
 			err = scenario.WaitForTailscaleSync()
-			assertNoErrSync(t, err)
+			requireNoErrSync(t, err)
 
 			headscale, err := scenario.Headscale()
-			assertNoErrGetHeadscale(t, err)
+			requireNoErrGetHeadscale(t, err)
 
-			expectedNodes := make([]types.NodeID, 0, len(allClients))
-			for _, client := range allClients {
-				status := client.MustStatus()
-				nodeID, err := strconv.ParseUint(string(status.Self.ID), 10, 64)
-				assertNoErr(t, err)
-				expectedNodes = append(expectedNodes, types.NodeID(nodeID))
-			}
-			requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", 30*time.Second)
+			expectedNodes := collectExpectedNodeIDs(t, allClients)
+			requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected", integrationutil.ScaledTimeout(120*time.Second))
 
-			// Validate that all nodes have NetInfo and DERP servers before logout
-			requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 1*time.Minute)
+			// Validate that all nodes have [tailcfg.NetInfo] and DERP servers before logout
+			requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP before logout", 3*time.Minute)
 
 			// assertClientsState(t, allClients)
 
 			clientIPs := make(map[TailscaleClient][]netip.Addr)
+
 			for _, client := range allClients {
 				ips, err := client.IPs()
 				if err != nil {
 					t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
 				}
+
 				clientIPs[client] = ips
 			}
 
-			listNodes, err := headscale.ListNodes()
-			assert.Len(t, allClients, len(listNodes))
-			nodeCountBeforeLogout := len(listNodes)
-			t.Logf("node count before logout: %d", nodeCountBeforeLogout)
+			var (
+				listNodes             []*v1.Node
+				nodeCountBeforeLogout int
+			)
 
-			for _, node := range listNodes {
-				assertLastSeenSet(t, node)
-			}
+			assert.EventuallyWithT(t, func(c *assert.CollectT) {
+				var err error
+
+				listNodes, err = headscale.ListNodes()
+				assert.NoError(c, err)
+				assert.Len(c, listNodes, len(allClients))
+
+				for _, node := range listNodes {
+					assertLastSeenSetWithCollect(c, node)
+				}
+			}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
+
+			nodeCountBeforeLogout = len(listNodes)
+			t.Logf("node count before logout: %d", nodeCountBeforeLogout)
 
 			for _, client := range allClients {
 				err := client.Logout()
@@ -97,19 +101,21 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 			}
 
 			err = scenario.WaitForTailscaleLogout()
-			assertNoErrLogout(t, err)
+			requireNoErrLogout(t, err)
 
 			// After taking down all nodes, verify all systems show nodes offline
-			requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", 120*time.Second)
+			requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should have logged out", integrationutil.ScaledTimeout(120*time.Second))
 
 			t.Logf("all clients logged out")
 
+			t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat))
 			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 				var err error
+
 				listNodes, err = headscale.ListNodes()
-				assert.NoError(ct, err)
-				assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count")
-			}, 20*time.Second, 1*time.Second)
+				assert.NoError(ct, err, "Failed to list nodes after logout")
+				assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
+			}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node persistence after logout (nodes should remain in database)")
 
 			for _, node := range listNodes {
 				assertLastSeenSet(t, node)
@@ -121,11 +127,12 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 			// https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38
 			// https://github.com/juanfont/headscale/issues/2164
 			if !https {
+				//nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS
 				time.Sleep(5 * time.Minute)
 			}
 
 			userMap, err := headscale.MapUsers()
-			assertNoErr(t, err)
+			require.NoError(t, err)
 
 			for _, userName := range spec.Users {
 				key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
@@ -139,31 +146,36 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 				}
 			}
 
+			t.Logf("Validating node persistence after relogin at %s", time.Now().Format(TimestampFormat))
 			assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 				var err error
+
 				listNodes, err = headscale.ListNodes()
-				assert.NoError(ct, err)
-				assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match after HTTPS reconnection")
-			}, 30*time.Second, 2*time.Second)
+				assert.NoError(ct, err, "Failed to list nodes after relogin")
+				assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after relogin - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
+			}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node count stability after same-user auth key relogin")
 
 			for _, node := range listNodes {
 				assertLastSeenSet(t, node)
 			}
 
-			requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", 120*time.Second)
+			requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected to batcher", integrationutil.ScaledTimeout(120*time.Second))
 
-			// Validate that all nodes have NetInfo and DERP servers after reconnection
-			requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 1*time.Minute)
+			// Wait for Tailscale sync before validating [tailcfg.NetInfo] to ensure proper state propagation
+			err = scenario.WaitForTailscaleSync()
+			requireNoErrSync(t, err)
+
+			// Validate that all nodes have [tailcfg.NetInfo] and DERP servers after reconnection
+			requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after reconnection", 3*time.Minute)
 
 			err = scenario.WaitForTailscaleSync()
-			assertNoErrSync(t, err)
+			requireNoErrSync(t, err)
 
 			allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
 				return x.String()
 			})
 
-			success := pingAllHelper(t, allClients, allAddrs)
-			t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+			assertPingAll(t, allClients, allAddrs)
 
 			for _, client := range allClients {
 				ips, err := client.IPs()
@@ -188,78 +200,25 @@ func TestAuthKeyLogoutAndReloginSameUser(t *testing.T) {
 				}
 			}
 
-			listNodes, err = headscale.ListNodes()
-			require.Len(t, listNodes, nodeCountBeforeLogout)
-			for _, node := range listNodes {
-				assertLastSeenSet(t, node)
-			}
+			assert.EventuallyWithT(t, func(c *assert.CollectT) {
+				var err error
+
+				listNodes, err = headscale.ListNodes()
+				assert.NoError(c, err)
+				assert.Len(c, listNodes, nodeCountBeforeLogout)
+
+				for _, node := range listNodes {
+					assertLastSeenSetWithCollect(c, node)
+				}
+			}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for node list after relogin")
 		})
 	}
 }
 
-// requireAllClientsNetInfoAndDERP validates that all nodes have NetInfo in the database
-// and a valid DERP server based on the NetInfo. This function follows the pattern of
-// requireAllClientsOnline by using hsic.DebugNodeStore to get the database state.
-func requireAllClientsNetInfoAndDERP(t *testing.T, headscale ControlServer, expectedNodes []types.NodeID, message string, timeout time.Duration) {
-	t.Helper()
-
-	startTime := time.Now()
-	t.Logf("requireAllClientsNetInfoAndDERP: Starting validation at %s - %s", startTime.Format(TimestampFormat), message)
-
-	require.EventuallyWithT(t, func(c *assert.CollectT) {
-		// Get nodestore state
-		nodeStore, err := headscale.DebugNodeStore()
-		assert.NoError(c, err, "Failed to get nodestore debug info")
-		if err != nil {
-			return
-		}
-
-		// Validate node counts first
-		expectedCount := len(expectedNodes)
-		assert.Equal(c, expectedCount, len(nodeStore), "NodeStore total nodes mismatch")
-
-		// Check each expected node
-		for _, nodeID := range expectedNodes {
-			node, exists := nodeStore[nodeID]
-			assert.True(c, exists, "Node %d not found in nodestore", nodeID)
-			if !exists {
-				continue
-			}
-
-			// Validate that the node has Hostinfo
-			assert.NotNil(c, node.Hostinfo, "Node %d (%s) should have Hostinfo", nodeID, node.Hostname)
-			if node.Hostinfo == nil {
-				continue
-			}
-
-			// Validate that the node has NetInfo
-			assert.NotNil(c, node.Hostinfo.NetInfo, "Node %d (%s) should have NetInfo in Hostinfo", nodeID, node.Hostname)
-			if node.Hostinfo.NetInfo == nil {
-				continue
-			}
-
-			// Validate that the node has a valid DERP server (PreferredDERP should be > 0)
-			preferredDERP := node.Hostinfo.NetInfo.PreferredDERP
-			assert.Greater(c, preferredDERP, 0, "Node %d (%s) should have a valid DERP server (PreferredDERP > 0), got %d", nodeID, node.Hostname, preferredDERP)
-
-			t.Logf("Node %d (%s) has valid NetInfo with DERP server %d", nodeID, node.Hostname, preferredDERP)
-		}
-	}, timeout, 2*time.Second, message)
-
-	endTime := time.Now()
-	duration := endTime.Sub(startTime)
-	t.Logf("requireAllClientsNetInfoAndDERP: Completed validation at %s - Duration: %v - %s", endTime.Format(TimestampFormat), duration, message)
-}
-
-func assertLastSeenSet(t *testing.T, node *v1.Node) {
-	assert.NotNil(t, node)
-	assert.NotNil(t, node.GetLastSeen())
-}
-
 // This test will first log in two sets of nodes to two sets of users, then
-// it will log out all users from user2 and log them in as user1.
-// This should leave us with all nodes connected to user1, while user2
-// still has nodes, but they are not connected.
+// it will log out all nodes and log them in as user1 using a pre-auth key.
+// This should create new nodes for user1 while preserving the original nodes for user2.
+// Pre-auth key re-authentication with a different user creates new nodes, not transfers.
 func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
 	IntegrationSkip(t)
 
@@ -269,30 +228,47 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+	require.NoError(t, err)
+
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	err = scenario.CreateHeadscaleEnv([]tsic.Option{},
 		hsic.WithTestName("keyrelognewuser"),
-		hsic.WithTLS(),
-		hsic.WithDERPAsIP(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// assertClientsState(t, allClients)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
 
-	listNodes, err := headscale.ListNodes()
-	assert.Len(t, allClients, len(listNodes))
-	nodeCountBeforeLogout := len(listNodes)
+	// Collect expected node IDs for validation
+	expectedNodes := collectExpectedNodeIDs(t, allClients)
+
+	// Validate initial connection state
+	requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second))
+	requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute)
+
+	var (
+		listNodes             []*v1.Node
+		nodeCountBeforeLogout int
+	)
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		var err error
+
+		listNodes, err = headscale.ListNodes()
+		assert.NoError(c, err)
+		assert.Len(c, listNodes, len(allClients))
+	}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
+
+	nodeCountBeforeLogout = len(listNodes)
 	t.Logf("node count before logout: %d", nodeCountBeforeLogout)
 
 	for _, client := range allClients {
@@ -303,12 +279,15 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
 	}
 
 	err = scenario.WaitForTailscaleLogout()
-	assertNoErrLogout(t, err)
+	requireNoErrLogout(t, err)
+
+	// Validate that all nodes are offline after logout
+	requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second))
 
 	t.Logf("all clients logged out")
 
 	userMap, err := headscale.MapUsers()
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	// Create a new authkey for user1, to be used for all clients
 	key, err := scenario.CreatePreAuthKey(userMap["user1"].GetId(), true, false)
@@ -326,28 +305,48 @@ func TestAuthKeyLogoutAndReloginNewUser(t *testing.T) {
 	}
 
 	var user1Nodes []*v1.Node
-	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-		var err error
-		user1Nodes, err = headscale.ListNodes("user1")
-		assert.NoError(ct, err)
-		assert.Len(ct, user1Nodes, len(allClients), "User1 should have all clients after re-login")
-	}, 20*time.Second, 1*time.Second)
 
-	// Validate that all the old nodes are still present with user2
-	var user2Nodes []*v1.Node
+	t.Logf("Validating user1 node count after relogin at %s", time.Now().Format(TimestampFormat))
 	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 		var err error
+
+		user1Nodes, err = headscale.ListNodes("user1")
+		assert.NoError(ct, err, "Failed to list nodes for user1 after relogin")
+		assert.Len(ct, user1Nodes, len(allClients), "User1 should have all %d clients after relogin, got %d nodes", len(allClients), len(user1Nodes))
+	}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating user1 has all client nodes after auth key relogin")
+
+	// Collect expected node IDs for user1 after relogin
+	expectedUser1Nodes := make([]types.NodeID, 0, len(user1Nodes))
+	for _, node := range user1Nodes {
+		expectedUser1Nodes = append(expectedUser1Nodes, types.NodeID(node.GetId()))
+	}
+
+	// Validate connection state after relogin as user1
+	requireAllClientsOnline(t, headscale, expectedUser1Nodes, true, "all user1 nodes should be connected after relogin", integrationutil.ScaledTimeout(120*time.Second))
+	requireAllClientsNetInfoAndDERP(t, headscale, expectedUser1Nodes, "all user1 nodes should have NetInfo and DERP after relogin", 3*time.Minute)
+
+	// Validate that user2 still has their original nodes after user1's re-authentication
+	// When nodes re-authenticate with a different user's pre-auth key, NEW nodes are created
+	// for the new user. The original nodes remain with the original user.
+	var user2Nodes []*v1.Node
+
+	t.Logf("Validating user2 node persistence after user1 relogin at %s", time.Now().Format(TimestampFormat))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+
 		user2Nodes, err = headscale.ListNodes("user2")
-		assert.NoError(ct, err)
-		assert.Len(ct, user2Nodes, len(allClients)/2, "User2 should have half the clients")
-	}, 20*time.Second, 1*time.Second)
+		assert.NoError(ct, err, "Failed to list nodes for user2 after user1 relogin")
+		assert.Len(ct, user2Nodes, len(allClients)/2, "User2 should still have %d clients after user1 relogin, got %d nodes", len(allClients)/2, len(user2Nodes))
+	}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating user2 nodes persist after user1 relogin (should not be affected)")
+
+	t.Logf("Validating client login states after user switch at %s", time.Now().Format(TimestampFormat))
 
 	for _, client := range allClients {
 		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 			status, err := client.Status()
 			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
-			assert.Equal(ct, "user1@test.no", status.User[status.Self.UserID].LoginName, "Client %s should be logged in as user1", client.Hostname())
-		}, 30*time.Second, 2*time.Second)
+			assert.Equal(ct, "user1@test.no", status.User[status.Self.UserID].LoginName, "Client %s should be logged in as user1 after user switch, got %s", client.Hostname(), status.User[status.Self.UserID].LoginName)
+		}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating %s is logged in as user1 after auth key user switch", client.Hostname())
 	}
 }
 
@@ -362,45 +361,60 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
 			}
 
 			scenario, err := NewScenario(spec)
-			assertNoErr(t, err)
+
+			require.NoError(t, err)
 			defer scenario.ShutdownAssertNoPanics(t)
 
 			opts := []hsic.Option{
-				hsic.WithTestName("pingallbyip"),
-				hsic.WithDERPAsIP(),
-			}
-			if https {
-				opts = append(opts, []hsic.Option{
-					hsic.WithTLS(),
-				}...)
+				hsic.WithTestName("authkey-rlogexpired"),
 			}
 
 			err = scenario.CreateHeadscaleEnv([]tsic.Option{}, opts...)
-			assertNoErrHeadscaleEnv(t, err)
+			requireNoErrHeadscaleEnv(t, err)
 
 			allClients, err := scenario.ListTailscaleClients()
-			assertNoErrListClients(t, err)
+			requireNoErrListClients(t, err)
 
 			err = scenario.WaitForTailscaleSync()
-			assertNoErrSync(t, err)
+			requireNoErrSync(t, err)
 
 			// assertClientsState(t, allClients)
 
 			clientIPs := make(map[TailscaleClient][]netip.Addr)
+
 			for _, client := range allClients {
 				ips, err := client.IPs()
 				if err != nil {
 					t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
 				}
+
 				clientIPs[client] = ips
 			}
 
 			headscale, err := scenario.Headscale()
-			assertNoErrGetHeadscale(t, err)
+			requireNoErrGetHeadscale(t, err)
 
-			listNodes, err := headscale.ListNodes()
-			assert.Len(t, allClients, len(listNodes))
-			nodeCountBeforeLogout := len(listNodes)
+			// Collect expected node IDs for validation
+			expectedNodes := collectExpectedNodeIDs(t, allClients)
+
+			// Validate initial connection state
+			requireAllClientsOnline(t, headscale, expectedNodes, true, "all clients should be connected after initial login", integrationutil.ScaledTimeout(120*time.Second))
+			requireAllClientsNetInfoAndDERP(t, headscale, expectedNodes, "all clients should have NetInfo and DERP after initial login", 3*time.Minute)
+
+			var (
+				listNodes             []*v1.Node
+				nodeCountBeforeLogout int
+			)
+
+			assert.EventuallyWithT(t, func(c *assert.CollectT) {
+				var err error
+
+				listNodes, err = headscale.ListNodes()
+				assert.NoError(c, err)
+				assert.Len(c, listNodes, len(allClients))
+			}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "Waiting for expected node list before logout")
+
+			nodeCountBeforeLogout = len(listNodes)
 			t.Logf("node count before logout: %d", nodeCountBeforeLogout)
 
 			for _, client := range allClients {
@@ -411,7 +425,10 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
 			}
 
 			err = scenario.WaitForTailscaleLogout()
-			assertNoErrLogout(t, err)
+			requireNoErrLogout(t, err)
+
+			// Validate that all nodes are offline after logout
+			requireAllClientsOnline(t, headscale, expectedNodes, false, "all nodes should be offline after logout", integrationutil.ScaledTimeout(120*time.Second))
 
 			t.Logf("all clients logged out")
 
@@ -421,11 +438,12 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
 			// https://github.com/tailscale/tailscale/commit/1eaad7d3deb0815e8932e913ca1a862afa34db38
 			// https://github.com/juanfont/headscale/issues/2164
 			if !https {
+				//nolint:forbidigo // Intentional delay: Tailscale client requires 5 min wait before reconnecting over non-HTTPS
 				time.Sleep(5 * time.Minute)
 			}
 
 			userMap, err := headscale.MapUsers()
-			assertNoErr(t, err)
+			require.NoError(t, err)
 
 			for _, userName := range spec.Users {
 				key, err := scenario.CreatePreAuthKey(userMap[userName].GetId(), true, false)
@@ -438,12 +456,12 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
 					[]string{
 						"headscale",
 						"preauthkeys",
-						"--user",
-						strconv.FormatUint(userMap[userName].GetId(), 10),
 						"expire",
-						key.GetKey(),
+						"--id",
+						strconv.FormatUint(key.GetId(), 10),
 					})
-				assertNoErr(t, err)
+				require.NoError(t, err)
+				require.NoError(t, err)
 
 				err = scenario.RunTailscaleUp(userName, headscale.GetEndpoint(), key.GetKey())
 				assert.ErrorContains(t, err, "authkey expired")
@@ -451,3 +469,280 @@ func TestAuthKeyLogoutAndReloginSameUserExpiredKey(t *testing.T) {
 		})
 	}
 }
+
+// TestAuthKeyDeleteKey tests Issue #2830: node with deleted auth key should still reconnect.
+// Scenario from user report: "create node, delete the auth key, restart to validate it can connect"
+// Steps:
+// 1. Create node with auth key
+// 2. DELETE the auth key from database (completely remove it)
+// 3. Restart node - should successfully reconnect using [tailcfg.Node.MachineKey] identity.
+func TestAuthKeyDeleteKey(t *testing.T) {
+	IntegrationSkip(t)
+
+	// Create scenario with NO nodes - we'll create the node manually so we can capture the auth key
+	scenario, err := NewScenario(ScenarioSpec{
+		NodesPerUser: 0, // No nodes created automatically
+		Users:        []string{"user1"},
+	})
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("delkey"))
+	requireNoErrHeadscaleEnv(t, err)
+
+	headscale, err := scenario.Headscale()
+	requireNoErrGetHeadscale(t, err)
+
+	// Get the user
+	userMap, err := headscale.MapUsers()
+	require.NoError(t, err)
+
+	userID := userMap["user1"].GetId()
+
+	// Create a pre-auth key - we keep the full key string before it gets redacted
+	authKey, err := scenario.CreatePreAuthKey(userID, false, false)
+	require.NoError(t, err)
+
+	authKeyString := authKey.GetKey()
+	authKeyID := authKey.GetId()
+	t.Logf("Created pre-auth key ID %d: %s", authKeyID, authKeyString)
+
+	// Create a tailscale client and log it in with the auth key
+	client, err := scenario.CreateTailscaleNode(
+		"head",
+		tsic.WithNetwork(scenario.networks[scenario.testDefaultNetwork]),
+	)
+	require.NoError(t, err)
+
+	err = client.Login(headscale.GetEndpoint(), authKeyString)
+	require.NoError(t, err)
+
+	// Wait for the node to be registered
+	var user1Nodes []*v1.Node
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		var err error
+
+		user1Nodes, err = headscale.ListNodes("user1")
+		assert.NoError(c, err)
+		assert.Len(c, user1Nodes, 1)
+	}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "waiting for node to be registered")
+
+	nodeID := user1Nodes[0].GetId()
+	nodeName := user1Nodes[0].GetName()
+	t.Logf("Node %d (%s) created successfully with auth_key_id=%d", nodeID, nodeName, authKeyID)
+
+	// Verify node is online
+	requireAllClientsOnline(t, headscale, []types.NodeID{types.NodeID(nodeID)}, true, "node should be online initially", integrationutil.ScaledTimeout(120*time.Second))
+
+	// DELETE the pre-auth key using the API
+	t.Logf("Deleting pre-auth key ID %d using API", authKeyID)
+
+	err = headscale.DeleteAuthKey(authKeyID)
+	require.NoError(t, err)
+	t.Logf("Successfully deleted auth key")
+
+	// Simulate node restart (down + up)
+	t.Logf("Restarting node after deleting its auth key")
+
+	err = client.Down()
+	require.NoError(t, err)
+
+	// Wait for client to fully stop before bringing it back up
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := client.Status()
+		assert.NoError(c, err)
+		assert.Equal(c, "Stopped", status.BackendState)
+	}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.FastPoll, "client should be stopped")
+
+	err = client.Up()
+	require.NoError(t, err)
+
+	// Verify node comes back online
+	// This will FAIL without the fix because auth key validation will reject deleted key
+	// With the fix, [tailcfg.Node.MachineKey] identity allows reconnection even with deleted key
+	requireAllClientsOnline(t, headscale, []types.NodeID{types.NodeID(nodeID)}, true, "node should reconnect after restart despite deleted key", integrationutil.ScaledTimeout(120*time.Second))
+
+	t.Logf("✓ Node successfully reconnected after its auth key was deleted")
+}
+
+// TestAuthKeyLogoutAndReloginRoutesPreserved tests that routes remain serving
+// after a node logs out and re-authenticates with the same user.
+//
+// This test validates the fix for issue #2896:
+// https://github.com/juanfont/headscale/issues/2896
+//
+// Bug: When a node with already-approved routes restarts/re-authenticates,
+// the routes show as "Approved" and "Available" but NOT "Serving" (Primary).
+// A headscale restart would fix it, indicating a state management issue.
+//
+// The test scenario:
+// 1. Node registers with auth key and advertises routes
+// 2. Routes are auto-approved and verified as serving
+// 3. Node logs out
+// 4. Node re-authenticates with same auth key
+// 5. Routes should STILL be serving (this is where the bug manifests).
+func TestAuthKeyLogoutAndReloginRoutesPreserved(t *testing.T) {
+	IntegrationSkip(t)
+
+	user := "routeuser"
+	advertiseRoute := "10.55.0.0/24"
+
+	spec := ScenarioSpec{
+		NodesPerUser: 1,
+		Users:        []string{user},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv(
+		[]tsic.Option{
+			tsic.WithAcceptRoutes(),
+			// Advertise route on initial login
+			tsic.WithExtraLoginArgs([]string{"--advertise-routes=" + advertiseRoute}),
+		},
+		hsic.WithTestName("routelogout"),
+		hsic.WithACLPolicy(
+			&policyv2.Policy{
+				ACLs: []policyv2.ACL{
+					{
+						Action:       "accept",
+						Sources:      []policyv2.Alias{policyv2.Wildcard},
+						Destinations: []policyv2.AliasWithPorts{{Alias: policyv2.Wildcard, Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}}},
+					},
+				},
+				AutoApprovers: policyv2.AutoApproverPolicy{
+					Routes: map[netip.Prefix]policyv2.AutoApprovers{
+						netip.MustParsePrefix(advertiseRoute): {new(policyv2.Username(user + "@test.no"))},
+					},
+				},
+			},
+		),
+	)
+	requireNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	requireNoErrListClients(t, err)
+	require.Len(t, allClients, 1)
+
+	client := allClients[0]
+
+	err = scenario.WaitForTailscaleSync()
+	requireNoErrSync(t, err)
+
+	headscale, err := scenario.Headscale()
+	requireNoErrGetHeadscale(t, err)
+
+	// Step 1: Verify initial route is advertised, approved, and SERVING
+	t.Logf("Step 1: Verifying initial route is advertised, approved, and SERVING at %s", time.Now().Format(TimestampFormat))
+
+	var initialNode *v1.Node
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		nodes, err := headscale.ListNodes()
+		assert.NoError(c, err)
+		assert.Len(c, nodes, 1, "Should have exactly 1 node")
+
+		if len(nodes) == 1 {
+			initialNode = nodes[0]
+			// Check: 1 announced, 1 approved, 1 serving (subnet route)
+			assert.Lenf(c, initialNode.GetAvailableRoutes(), 1,
+				"Node should have 1 available route, got %v", initialNode.GetAvailableRoutes())
+			assert.Lenf(c, initialNode.GetApprovedRoutes(), 1,
+				"Node should have 1 approved route, got %v", initialNode.GetApprovedRoutes())
+			assert.Lenf(c, initialNode.GetSubnetRoutes(), 1,
+				"Node should have 1 serving (subnet) route, got %v - THIS IS THE BUG if empty", initialNode.GetSubnetRoutes())
+			assert.Contains(c, initialNode.GetSubnetRoutes(), advertiseRoute,
+				"Subnet routes should contain %s", advertiseRoute)
+		}
+	}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll, "initial route should be serving")
+
+	require.NotNil(t, initialNode, "Initial node should be found")
+	initialNodeID := initialNode.GetId()
+	t.Logf("Initial node ID: %d, Available: %v, Approved: %v, Serving: %v",
+		initialNodeID, initialNode.GetAvailableRoutes(), initialNode.GetApprovedRoutes(), initialNode.GetSubnetRoutes())
+
+	// Step 2: Logout
+	t.Logf("Step 2: Logging out at %s", time.Now().Format(TimestampFormat))
+
+	err = client.Logout()
+	require.NoError(t, err)
+
+	// Wait for logout to complete
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		status, err := client.Status()
+		assert.NoError(ct, err)
+		assert.Equal(ct, "NeedsLogin", status.BackendState, "Expected NeedsLogin state after logout")
+	}, integrationutil.StatusReadyTimeout, 1*time.Second, "waiting for logout to complete")
+
+	t.Logf("Logout completed, node should still exist in database")
+
+	// Verify node still exists (routes should still be in DB)
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		nodes, err := headscale.ListNodes()
+		assert.NoError(c, err)
+		assert.Len(c, nodes, 1, "Node should persist in database after logout")
+	}, integrationutil.ScaledTimeout(10*time.Second), integrationutil.SlowPoll, "node should persist after logout")
+
+	// Step 3: Re-authenticate with the SAME user (using auth key)
+	t.Logf("Step 3: Re-authenticating with same user at %s", time.Now().Format(TimestampFormat))
+
+	userMap, err := headscale.MapUsers()
+	require.NoError(t, err)
+
+	key, err := scenario.CreatePreAuthKey(userMap[user].GetId(), true, false)
+	require.NoError(t, err)
+
+	// Re-login - the container already has extraLoginArgs with --advertise-routes
+	// from the initial setup, so routes will be advertised on re-login
+	err = scenario.RunTailscaleUp(user, headscale.GetEndpoint(), key.GetKey())
+	require.NoError(t, err)
+
+	// Wait for client to be running
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		status, err := client.Status()
+		assert.NoError(ct, err)
+		assert.Equal(ct, "Running", status.BackendState, "Expected Running state after relogin")
+	}, integrationutil.StatusReadyTimeout, 1*time.Second, "waiting for relogin to complete")
+
+	t.Logf("Re-authentication completed at %s", time.Now().Format(TimestampFormat))
+
+	// Step 4: THE CRITICAL TEST - Verify routes are STILL SERVING after re-authentication
+	t.Logf("Step 4: Verifying routes are STILL SERVING after re-authentication at %s", time.Now().Format(TimestampFormat))
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		nodes, err := headscale.ListNodes()
+		assert.NoError(c, err)
+		assert.Len(c, nodes, 1, "Should still have exactly 1 node after relogin")
+
+		if len(nodes) == 1 {
+			node := nodes[0]
+			t.Logf("After relogin - Available: %v, Approved: %v, Serving: %v",
+				node.GetAvailableRoutes(), node.GetApprovedRoutes(), node.GetSubnetRoutes())
+
+			// This is where issue #2896 manifests:
+			// - Available shows the route (from [tailcfg.Hostinfo.RoutableIPs])
+			// - Approved shows the route (from [tailcfg.Node.ApprovedRoutes])
+			// - BUT Serving ([tailcfg.Node.SubnetRoutes]/[ipnstate.PeerStatus.PrimaryRoutes]) is EMPTY!
+			assert.Lenf(c, node.GetAvailableRoutes(), 1,
+				"Node should have 1 available route after relogin, got %v", node.GetAvailableRoutes())
+			assert.Lenf(c, node.GetApprovedRoutes(), 1,
+				"Node should have 1 approved route after relogin, got %v", node.GetApprovedRoutes())
+			assert.Lenf(c, node.GetSubnetRoutes(), 1,
+				"BUG #2896: Node should have 1 SERVING route after relogin, got %v", node.GetSubnetRoutes())
+			assert.Contains(c, node.GetSubnetRoutes(), advertiseRoute,
+				"BUG #2896: Subnet routes should contain %s after relogin", advertiseRoute)
+
+			// Also verify node ID was preserved (same node, not new registration)
+			assert.Equal(c, initialNodeID, node.GetId(),
+				"Node ID should be preserved after same-user relogin")
+		}
+	}, integrationutil.StatusReadyTimeout, integrationutil.SlowPoll,
+		"BUG #2896: routes should remain SERVING after logout/relogin with same user")
+
+	t.Logf("Test completed - verifying issue #2896 fix")
+}

integration/auth_web_flow_test.go

@@ -7,9 +7,12 @@ import (
 	"time"
 
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
+	"github.com/juanfont/headscale/hscontrol/types"
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/samber/lo"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
@@ -29,20 +32,17 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 	err = scenario.CreateHeadscaleEnvWithLoginURL(
 		nil,
 		hsic.WithTestName("webauthping"),
-		hsic.WithEmbeddedDERPServerOnly(),
-		hsic.WithDERPAsIP(),
-		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	allIps, err := scenario.ListTailscaleClientsIPs()
-	assertNoErrListClientIPs(t, err)
+	requireNoErrListClientIPs(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// assertClientsState(t, allClients)
 
@@ -50,11 +50,10 @@ func TestAuthWebFlowAuthenticationPingAll(t *testing.T) {
 		return x.String()
 	})
 
-	success := pingAllHelper(t, allClients, allAddrs)
-	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+	assertPingAll(t, allClients, allAddrs)
 }
 
-func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
+func TestAuthWebFlowLogoutAndReloginSameUser(t *testing.T) {
 	IntegrationSkip(t)
 
 	spec := ScenarioSpec{
@@ -63,25 +62,24 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	err = scenario.CreateHeadscaleEnvWithLoginURL(
 		nil,
 		hsic.WithTestName("weblogout"),
-		hsic.WithDERPAsIP(),
-		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	allIps, err := scenario.ListTailscaleClientsIPs()
-	assertNoErrListClientIPs(t, err)
+	requireNoErrListClientIPs(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// assertClientsState(t, allClients)
 
@@ -89,28 +87,39 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 		return x.String()
 	})
 
-	success := pingAllHelper(t, allClients, allAddrs)
-	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
+	assertPingAll(t, allClients, allAddrs)
 
 	headscale, err := scenario.Headscale()
-	assertNoErrGetHeadscale(t, err)
+	requireNoErrGetHeadscale(t, err)
+
+	// Collect expected node IDs for validation
+	expectedNodes := collectExpectedNodeIDs(t, allClients)
+
+	// Validate initial connection state
+	validateInitialConnection(t, headscale, expectedNodes)
 
 	var listNodes []*v1.Node
+
+	t.Logf("Validating initial node count after web auth at %s", time.Now().Format(TimestampFormat))
 	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
 		var err error
+
 		listNodes, err = headscale.ListNodes()
-		assert.NoError(ct, err)
-		assert.Len(ct, listNodes, len(allClients), "Node count should match client count after login")
-	}, 20*time.Second, 1*time.Second)
+		assert.NoError(ct, err, "Failed to list nodes after web authentication")
+		assert.Len(ct, listNodes, len(allClients), "Expected %d nodes after web auth, got %d", len(allClients), len(listNodes))
+	}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node count matches client count after web authentication")
+
 	nodeCountBeforeLogout := len(listNodes)
 	t.Logf("node count before logout: %d", nodeCountBeforeLogout)
 
 	clientIPs := make(map[TailscaleClient][]netip.Addr)
+
 	for _, client := range allClients {
 		ips, err := client.IPs()
 		if err != nil {
 			t.Fatalf("failed to get IPs for client %s: %s", client.Hostname(), err)
 		}
+
 		clientIPs[client] = ips
 	}
 
@@ -122,7 +131,10 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 	}
 
 	err = scenario.WaitForTailscaleLogout()
-	assertNoErrLogout(t, err)
+	requireNoErrLogout(t, err)
+
+	// Validate that all nodes are offline after logout
+	validateLogoutComplete(t, headscale, expectedNodes)
 
 	t.Logf("all clients logged out")
 
@@ -135,23 +147,27 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 
 	t.Logf("all clients logged in again")
 
+	t.Logf("Validating node persistence after logout at %s", time.Now().Format(TimestampFormat))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+
+		listNodes, err = headscale.ListNodes()
+		assert.NoError(ct, err, "Failed to list nodes after web flow logout")
+		assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should remain unchanged after logout - expected %d nodes, got %d", nodeCountBeforeLogout, len(listNodes))
+	}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating node persistence in database after web flow logout")
+	t.Logf("node count first login: %d, after relogin: %d", nodeCountBeforeLogout, len(listNodes))
+
+	// Validate connection state after relogin
+	validateReloginComplete(t, headscale, expectedNodes)
+
 	allIps, err = scenario.ListTailscaleClientsIPs()
-	assertNoErrListClientIPs(t, err)
+	requireNoErrListClientIPs(t, err)
 
 	allAddrs = lo.Map(allIps, func(x netip.Addr, index int) string {
 		return x.String()
 	})
 
-	success = pingAllHelper(t, allClients, allAddrs)
-	t.Logf("%d successful pings out of %d", success, len(allClients)*len(allIps))
-
-	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
-		var err error
-		listNodes, err = headscale.ListNodes()
-		assert.NoError(ct, err)
-		assert.Len(ct, listNodes, nodeCountBeforeLogout, "Node count should match before logout count after re-login")
-	}, 20*time.Second, 1*time.Second)
-	t.Logf("node count first login: %d, after relogin: %d", nodeCountBeforeLogout, len(listNodes))
+	assertPingAll(t, allClients, allAddrs)
 
 	for _, client := range allClients {
 		ips, err := client.IPs()
@@ -180,3 +196,176 @@ func TestAuthWebFlowLogoutAndRelogin(t *testing.T) {
 
 	t.Logf("all clients IPs are the same")
 }
+
+// TestAuthWebFlowLogoutAndReloginNewUser tests the scenario where multiple Tailscale clients
+// initially authenticate using the web-based authentication flow (where users visit a URL
+// in their browser to authenticate), then all clients log out and log back in as a different user.
+//
+// This test validates the "user switching" behavior in headscale's web authentication flow:
+// - Multiple clients authenticate via web flow, each to their respective users (user1, user2)
+// - All clients log out simultaneously
+// - All clients log back in via web flow, but this time they all authenticate as user1
+// - The test verifies that user1 ends up with all the client nodes
+// - The test verifies that user2's original nodes still exist in the database but are offline
+// - The test verifies network connectivity works after the user switch
+//
+// This scenario is important for organizations that need to reassign devices between users
+// or when consolidating multiple user accounts. It ensures that headscale properly handles
+// the security implications of user switching while maintaining node persistence in the database.
+//
+// The test uses headscale's web authentication flow, which is the most user-friendly method
+// where authentication happens through a web browser rather than pre-shared keys or OIDC.
+func TestAuthWebFlowLogoutAndReloginNewUser(t *testing.T) {
+	IntegrationSkip(t)
+
+	spec := ScenarioSpec{
+		NodesPerUser: len(MustTestVersions),
+		Users:        []string{"user1", "user2"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnvWithLoginURL(
+		nil,
+		hsic.WithTestName("webflowrelnewuser"),
+	)
+	requireNoErrHeadscaleEnv(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	requireNoErrListClients(t, err)
+
+	var allIps []netip.Addr
+
+	allIps, err = scenario.ListTailscaleClientsIPs()
+	requireNoErrListClientIPs(t, err)
+
+	_ = allIps // used below after user switch
+
+	err = scenario.WaitForTailscaleSync()
+	requireNoErrSync(t, err)
+
+	headscale, err := scenario.Headscale()
+	requireNoErrGetHeadscale(t, err)
+
+	// Collect expected node IDs for validation
+	expectedNodes := collectExpectedNodeIDs(t, allClients)
+
+	// Validate initial connection state
+	validateInitialConnection(t, headscale, expectedNodes)
+
+	var listNodes []*v1.Node
+
+	t.Logf("Validating initial node count after web auth at %s", time.Now().Format(TimestampFormat))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+
+		listNodes, err = headscale.ListNodes()
+		assert.NoError(ct, err, "Failed to list nodes after initial web authentication")
+		assert.Len(ct, listNodes, len(allClients), "Expected %d nodes after web auth, got %d", len(allClients), len(listNodes))
+	}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating node count matches client count after initial web authentication")
+
+	nodeCountBeforeLogout := len(listNodes)
+	t.Logf("node count before logout: %d", nodeCountBeforeLogout)
+
+	// Log out all clients
+	for _, client := range allClients {
+		err := client.Logout()
+		if err != nil {
+			t.Fatalf("failed to logout client %s: %s", client.Hostname(), err)
+		}
+	}
+
+	err = scenario.WaitForTailscaleLogout()
+	requireNoErrLogout(t, err)
+
+	// Validate that all nodes are offline after logout
+	validateLogoutComplete(t, headscale, expectedNodes)
+
+	t.Logf("all clients logged out")
+
+	// Log all clients back in as user1 using web flow
+	// We manually iterate over all clients and authenticate each one as user1
+	// This tests the cross-user re-authentication behavior where ALL clients
+	// (including those originally from user2) are registered to user1
+	for _, client := range allClients {
+		loginURL, err := client.LoginWithURL(headscale.GetEndpoint())
+		if err != nil {
+			t.Fatalf("failed to get login URL for client %s: %s", client.Hostname(), err)
+		}
+
+		body, err := doLoginURL(client.Hostname(), loginURL)
+		if err != nil {
+			t.Fatalf("failed to complete login for client %s: %s", client.Hostname(), err)
+		}
+
+		// Register all clients as user1 (this is where cross-user registration happens)
+		// This simulates: headscale auth register --auth-id <id> --user user1
+		_ = scenario.runHeadscaleRegister("user1", body)
+	}
+
+	// Wait for all clients to reach running state
+	for _, client := range allClients {
+		err := client.WaitForRunning(integrationutil.PeerSyncTimeout())
+		if err != nil {
+			t.Fatalf("%s tailscale node has not reached running: %s", client.Hostname(), err)
+		}
+	}
+
+	t.Logf("all clients logged back in as user1")
+
+	var user1Nodes []*v1.Node
+
+	t.Logf("Validating user1 node count after relogin at %s", time.Now().Format(TimestampFormat))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+
+		user1Nodes, err = headscale.ListNodes("user1")
+		assert.NoError(ct, err, "Failed to list nodes for user1 after web flow relogin")
+		assert.Len(ct, user1Nodes, len(allClients), "User1 should have all %d clients after web flow relogin, got %d nodes", len(allClients), len(user1Nodes))
+	}, integrationutil.HAConvergeTimeout, 2*time.Second, "validating user1 has all client nodes after web flow user switch relogin")
+
+	// Collect expected node IDs for user1 after relogin
+	expectedUser1Nodes := make([]types.NodeID, 0, len(user1Nodes))
+	for _, node := range user1Nodes {
+		expectedUser1Nodes = append(expectedUser1Nodes, types.NodeID(node.GetId()))
+	}
+
+	// Validate connection state after relogin as user1
+	validateReloginComplete(t, headscale, expectedUser1Nodes)
+
+	// Validate that user2's old nodes still exist in database (but are expired/offline)
+	// When CLI registration creates new nodes for user1, user2's old nodes remain
+	var user2Nodes []*v1.Node
+
+	t.Logf("Validating user2 old nodes remain in database after CLI registration to user1 at %s", time.Now().Format(TimestampFormat))
+	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+		var err error
+
+		user2Nodes, err = headscale.ListNodes("user2")
+		assert.NoError(ct, err, "Failed to list nodes for user2 after CLI registration to user1")
+		assert.Len(ct, user2Nodes, len(allClients)/2, "User2 should still have %d old nodes (likely expired) after CLI registration to user1, got %d nodes", len(allClients)/2, len(user2Nodes))
+	}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating user2 old nodes remain in database after CLI registration to user1")
+
+	t.Logf("Validating client login states after web flow user switch at %s", time.Now().Format(TimestampFormat))
+
+	for _, client := range allClients {
+		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
+			status, err := client.Status()
+			assert.NoError(ct, err, "Failed to get status for client %s", client.Hostname())
+			assert.Equal(ct, "user1@test.no", status.User[status.Self.UserID].LoginName, "Client %s should be logged in as user1 after web flow user switch, got %s", client.Hostname(), status.User[status.Self.UserID].LoginName)
+		}, integrationutil.StatusReadyTimeout, 2*time.Second, "validating %s is logged in as user1 after web flow user switch", client.Hostname())
+	}
+
+	// Test connectivity after user switch
+	allIps, err = scenario.ListTailscaleClientsIPs()
+	requireNoErrListClientIPs(t, err)
+
+	allAddrs := lo.Map(allIps, func(x netip.Addr, index int) string {
+		return x.String()
+	})
+
+	assertPingAll(t, allClients, allAddrs)
+}

integration/cli_policy_test.go

@@ -0,0 +1,280 @@
+package integration
+
+import (
+	"encoding/json"
+	"testing"
+
+	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+)
+
+// TestPolicyCheckCommand exercises `headscale policy check` across the
+// matrix that nblock asked about on PR #3229:
+//
+//   - policyMode: server runs with policy_mode=file vs policy_mode=database.
+//     `check` reads from `--file`, so the server-side mode should not
+//     change the outcome; running both proves that.
+//   - fixture: ACL only, ACL with passing tests, ACL with failing tests.
+//   - bypass: no-bypass talks to the server over gRPC; bypass opens the
+//     database directly.
+//
+// Each row spins up its own scenario because policy_mode is fixed at boot
+// via `HEADSCALE_POLICY_MODE`. The two users + two nodes give the tests
+// block real `user@` aliases to resolve against.
+func TestPolicyCheckCommand(t *testing.T) {
+	IntegrationSkip(t)
+
+	type fixture struct {
+		name   string
+		policy policyv2.Policy
+	}
+
+	const (
+		user1 = "user1@"
+		user2 = "user2@"
+	)
+
+	aclOnly := policyv2.Policy{
+		ACLs: []policyv2.ACL{
+			{
+				Action:   policyv2.ActionAccept,
+				Protocol: "tcp", //nolint:goconst // protocol literal, used inline once
+				Sources:  []policyv2.Alias{usernamep(user1)},
+				Destinations: []policyv2.AliasWithPorts{
+					aliasWithPorts(usernamep(user2), tailcfg.PortRange{First: 22, Last: 22}),
+				},
+			},
+		},
+	}
+
+	aclPlusPassingTests := aclOnly
+	aclPlusPassingTests.Tests = []policyv2.PolicyTest{
+		{
+			Src:    user1,
+			Accept: []string{user2 + ":22"},
+		},
+	}
+
+	aclPlusFailingTests := aclOnly
+	aclPlusFailingTests.Tests = []policyv2.PolicyTest{
+		{
+			// Reverse direction is not allowed by the ACL; the test
+			// asserts ALLOWED, so it must fail.
+			Src:    user2,
+			Accept: []string{user1 + ":22"},
+		},
+	}
+
+	fixtures := []fixture{
+		{name: "acl-only", policy: aclOnly},
+		{name: "acl-plus-passing-tests", policy: aclPlusPassingTests},
+		{name: "acl-plus-failing-tests", policy: aclPlusFailingTests},
+	}
+
+	type row struct {
+		name       string
+		policyMode string
+		fixture    fixture
+		bypass     bool
+		wantErr    string
+		wantStdout string
+	}
+
+	modes := []string{"file", "database"} //nolint:goconst // axis labels match HEADSCALE_POLICY_MODE values
+	bypasses := []bool{false, true}
+	rows := make([]row, 0, len(modes)*len(fixtures)*len(bypasses))
+
+	for _, mode := range modes {
+		for _, f := range fixtures {
+			for _, bypass := range bypasses {
+				suffix := "no-bypass"
+				if bypass {
+					suffix = "bypass"
+				}
+
+				r := row{
+					name:       mode + "-" + f.name + "-" + suffix,
+					policyMode: mode,
+					fixture:    f,
+					bypass:     bypass,
+					wantStdout: "Policy is valid",
+				}
+				if f.name == "acl-plus-failing-tests" {
+					r.wantErr = "test(s) failed"
+					r.wantStdout = ""
+				}
+
+				rows = append(rows, r)
+			}
+		}
+	}
+
+	for _, tt := range rows {
+		t.Run(tt.name, func(t *testing.T) {
+			spec := ScenarioSpec{
+				NodesPerUser: 1,
+				Users:        []string{"user1", "user2"}, //nolint:goconst // matches usernamep("user1@")/("user2@") above
+			}
+
+			scenario, err := NewScenario(spec)
+			require.NoError(t, err)
+
+			defer scenario.ShutdownAssertNoPanics(t)
+
+			err = scenario.CreateHeadscaleEnv(
+				[]tsic.Option{},
+				hsic.WithTestName("cli-policycheck"),
+				hsic.WithConfigEnv(map[string]string{
+					"HEADSCALE_POLICY_MODE": tt.policyMode, //nolint:goconst // env var name from hscontrol/types/config.go
+				}),
+			)
+			require.NoError(t, err)
+
+			headscale, err := scenario.Headscale()
+			require.NoError(t, err)
+
+			pBytes, err := json.Marshal(tt.fixture.policy)
+			require.NoError(t, err)
+
+			policyFilePath := "/etc/headscale/policy.json" //nolint:goconst // standard headscale policy path
+			err = headscale.WriteFile(policyFilePath, pBytes)
+			require.NoError(t, err)
+
+			cmd := []string{"headscale", "policy", "check", "-f", policyFilePath} //nolint:goconst // CLI invocation
+			if tt.bypass {
+				// --force suppresses the "is the server running?"
+				// confirmation prompt so the command can run
+				// non-interactively under the test harness.
+				cmd = append(cmd, "--bypass-grpc-and-access-database-directly", "--force")
+			}
+
+			stdout, err := headscale.Execute(cmd)
+
+			if tt.wantErr != "" {
+				require.ErrorContains(t, err, tt.wantErr)
+
+				return
+			}
+
+			require.NoError(t, err)
+			require.Contains(t, stdout, tt.wantStdout)
+		})
+	}
+}
+
+// TestSSHTestsRejectFailingPolicy asserts `headscale policy set` rejects
+// a policy whose sshTests fail, surfaces the engine's "test(s) failed"
+// sentinel, and leaves the stored policy unchanged. autogroup:member as
+// dst lets every scenario node count, so no tagged node is needed.
+func TestSSHTestsRejectFailingPolicy(t *testing.T) {
+	IntegrationSkip(t)
+
+	const (
+		user1 = "user1@"
+		user2 = "user2@"
+	)
+
+	// Good policy: user1@ may SSH as root, and the sshTests asserts it.
+	goodPolicy := policyv2.Policy{
+		SSHs: []policyv2.SSH{
+			{
+				Action:  policyv2.SSHActionAccept,
+				Sources: policyv2.SSHSrcAliases{usernamep(user1)},
+				Destinations: policyv2.SSHDstAliases{
+					new(policyv2.AutoGroupMember),
+				},
+				Users: []policyv2.SSHUser{policyv2.SSHUser("root")},
+			},
+		},
+		SSHTests: []policyv2.SSHPolicyTest{
+			{
+				Src:    usernamep(user1),
+				Dst:    policyv2.SSHTestDestinations{new(policyv2.AutoGroupMember)},
+				Accept: []policyv2.SSHUser{policyv2.SSHUser("root")},
+			},
+		},
+	}
+
+	// Bad policy: same SSH rule, but the sshTests asserts user2@ — who
+	// the rule does not admit — can SSH. Must be rejected.
+	badPolicy := goodPolicy
+	badPolicy.SSHTests = []policyv2.SSHPolicyTest{
+		{
+			Src:    usernamep(user2),
+			Dst:    policyv2.SSHTestDestinations{new(policyv2.AutoGroupMember)},
+			Accept: []policyv2.SSHUser{policyv2.SSHUser("root")},
+		},
+	}
+
+	spec := ScenarioSpec{
+		NodesPerUser: 1,
+		Users:        []string{"user1", "user2"},
+	}
+
+	scenario, err := NewScenario(spec)
+	require.NoError(t, err)
+
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv(
+		[]tsic.Option{},
+		hsic.WithTestName("cli-policyset-sshtests"),
+		hsic.WithConfigEnv(map[string]string{
+			"HEADSCALE_POLICY_MODE": types.PolicyModeDB,
+		}),
+	)
+	require.NoError(t, err)
+
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	goodBytes, err := json.Marshal(goodPolicy)
+	require.NoError(t, err)
+
+	badBytes, err := json.Marshal(badPolicy)
+	require.NoError(t, err)
+
+	const (
+		goodPath = "/etc/headscale/policy-good.json"
+		badPath  = "/etc/headscale/policy-bad.json"
+	)
+
+	require.NoError(t, headscale.WriteFile(goodPath, goodBytes))
+	require.NoError(t, headscale.WriteFile(badPath, badBytes))
+
+	// Establish the good policy as the live policy.
+	_, err = headscale.Execute([]string{
+		"headscale", "policy", "set", "-f", goodPath,
+	})
+	require.NoError(t, err, "setting the good policy must succeed")
+
+	// Confirm the server returns the good policy.
+	stdoutBefore, err := headscale.Execute([]string{
+		"headscale", "policy", "get",
+	})
+	require.NoError(t, err)
+	require.JSONEq(t, string(goodBytes), stdoutBefore,
+		"server should report the good policy after the initial set")
+
+	// Attempt to overwrite with a policy whose sshTests fail. The CLI
+	// must surface the engine's "test(s) failed" sentinel and exit
+	// non-zero.
+	_, err = headscale.Execute([]string{
+		"headscale", "policy", "set", "-f", badPath,
+	})
+	require.Error(t, err, "setting a policy with failing sshTests must fail")
+	require.ErrorContains(t, err, "test(s) failed",
+		"CLI error must surface the engine's test failure sentinel")
+
+	// The rejected write must not have mutated the stored policy.
+	stdoutAfter, err := headscale.Execute([]string{
+		"headscale", "policy", "get",
+	})
+	require.NoError(t, err)
+	require.JSONEq(t, string(goodBytes), stdoutAfter,
+		"stored policy must be unchanged after a rejected set")
+}

integration/control.go

@@ -6,16 +6,17 @@ import (
 	v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
 	"github.com/juanfont/headscale/hscontrol"
 	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
-	"github.com/juanfont/headscale/hscontrol/routes"
 	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/ory/dockertest/v3"
 	"tailscale.com/tailcfg"
 )
 
 type ControlServer interface {
 	Shutdown() (string, string, error)
-	SaveLog(string) (string, string, error)
-	SaveProfile(string) error
+	SaveLog(path string) (string, string, error)
+	ReadLog() (string, string, error)
+	SaveProfile(path string) error
 	Execute(command []string) (string, error)
 	WriteFile(path string, content []byte) error
 	ConnectToNetwork(network *dockertest.Network) error
@@ -24,18 +25,25 @@ type ControlServer interface {
 	WaitForRunning() error
 	CreateUser(user string) (*v1.User, error)
 	CreateAuthKey(user uint64, reusable bool, ephemeral bool) (*v1.PreAuthKey, error)
+	CreateAuthKeyWithTags(user uint64, reusable bool, ephemeral bool, tags []string) (*v1.PreAuthKey, error)
+	CreateAuthKeyWithOptions(opts hsic.AuthKeyOptions) (*v1.PreAuthKey, error)
+	DeleteAuthKey(id uint64) error
 	ListNodes(users ...string) ([]*v1.Node, error)
+	DeleteNode(nodeID uint64) error
 	NodesByUser() (map[string][]*v1.Node, error)
 	NodesByName() (map[string]*v1.Node, error)
 	ListUsers() ([]*v1.User, error)
 	MapUsers() (map[string]*v1.User, error)
-	ApproveRoutes(uint64, []netip.Prefix) (*v1.Node, error)
+	DeleteUser(userID uint64) error
+	ApproveRoutes(nodeID uint64, routes []netip.Prefix) (*v1.Node, error)
+	SetNodeTags(nodeID uint64, tags []string) error
 	GetCert() []byte
 	GetHostname() string
 	GetIPInNetwork(network *dockertest.Network) string
-	SetPolicy(*policyv2.Policy) error
+	SetPolicy(pol *policyv2.Policy) error
 	GetAllMapReponses() (map[types.NodeID][]tailcfg.MapResponse, error)
-	PrimaryRoutes() (*routes.DebugRoutes, error)
+	PrimaryRoutes() (*types.DebugRoutes, error)
 	DebugBatcher() (*hscontrol.DebugBatcherInfo, error)
 	DebugNodeStore() (map[types.NodeID]types.Node, error)
+	DebugFilter() ([]tailcfg.FilterRule, error)
 }

integration/derp_verify_endpoint_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/require"
 	"tailscale.com/derp"
 	"tailscale.com/derp/derphttp"
 	"tailscale.com/net/netmon"
@@ -23,15 +24,16 @@ func TestDERPVerifyEndpoint(t *testing.T) {
 
 	// Generate random hostname for the headscale instance
 	hash, err := util.GenerateRandomStringDNSSafe(6)
-	assertNoErr(t, err)
+	require.NoError(t, err)
+
 	testName := "derpverify"
 	hostname := fmt.Sprintf("hs-%s-%s", testName, hash)
 
 	headscalePort := 8080
 
 	// Create cert for headscale
-	certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname)
-	assertNoErr(t, err)
+	caHeadscale, certHeadscale, keyHeadscale, err := integrationutil.CreateCertificate(hostname)
+	require.NoError(t, err)
 
 	spec := ScenarioSpec{
 		NodesPerUser: len(MustTestVersions),
@@ -39,14 +41,15 @@ func TestDERPVerifyEndpoint(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	derper, err := scenario.CreateDERPServer("head",
-		dsic.WithCACert(certHeadscale),
+		dsic.WithCACert(caHeadscale),
 		dsic.WithVerifyClientURL(fmt.Sprintf("https://%s/verify", net.JoinHostPort(hostname, strconv.Itoa(headscalePort)))),
 	)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	derpRegion := tailcfg.DERPRegion{
 		RegionCode: "test-derpverify",
@@ -69,22 +72,30 @@ func TestDERPVerifyEndpoint(t *testing.T) {
 		},
 	}
 
+	// [hsic.WithHostname] is used instead of [hsic.WithTestName] because the hostname
+	// must match the pre-generated TLS certificate created above.
+	// The test name "derpverify" is embedded in the hostname variable.
+	//
+	// [tsic.WithCACert] passes the external DERP server's certificate so
+	// tailscale clients trust it. [hsic.WithCustomTLS] and [hsic.WithDERPConfig]
+	// configure headscale to use the external DERP server created
+	// above instead of the default embedded one.
 	err = scenario.CreateHeadscaleEnv([]tsic.Option{tsic.WithCACert(derper.GetCert())},
 		hsic.WithHostname(hostname),
 		hsic.WithPort(headscalePort),
-		hsic.WithCustomTLS(certHeadscale, keyHeadscale),
+		hsic.WithCustomTLS(caHeadscale, certHeadscale, keyHeadscale),
 		hsic.WithDERPConfig(derpMap))
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	fakeKey := key.NewNode()
 	DERPVerify(t, fakeKey, derpRegion, false)
 
 	for _, client := range allClients {
 		nodeKey, err := client.GetNodePrivateKey()
-		assertNoErr(t, err)
+		require.NoError(t, err)
 		DERPVerify(t, *nodeKey, derpRegion, true)
 	}
 }
@@ -103,13 +114,16 @@ func DERPVerify(
 	defer c.Close()
 
 	var result error
-	if err := c.Connect(t.Context()); err != nil {
+
+	err := c.Connect(t.Context())
+	if err != nil {
 		result = fmt.Errorf("client Connect: %w", err)
 	}
-	if m, err := c.Recv(); err != nil {
+
+	if m, err := c.Recv(); err != nil { //nolint:noinlineerr
 		result = fmt.Errorf("client first Recv: %w", err)
 	} else if v, ok := m.(derp.ServerInfoMessage); !ok {
-		result = fmt.Errorf("client first Recv was unexpected type %T", v)
+		result = fmt.Errorf("client first Recv was unexpected type %T", v) //nolint:err113
 	}
 
 	if expectSuccess && result != nil {

integration/dns_test.go

@@ -8,8 +8,10 @@ import (
 	"time"
 
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"tailscale.com/tailcfg"
 )
 
@@ -22,26 +24,27 @@ func TestResolveMagicDNS(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	err = scenario.CreateHeadscaleEnv([]tsic.Option{}, hsic.WithTestName("magicdns"))
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// assertClientsState(t, allClients)
 
 	// Poor mans cache
 	_, err = scenario.ListTailscaleClientsFQDNs()
-	assertNoErrListFQDN(t, err)
+	requireNoErrListFQDN(t, err)
 
 	_, err = scenario.ListTailscaleClientsIPs()
-	assertNoErrListClientIPs(t, err)
+	requireNoErrListClientIPs(t, err)
 
 	for _, client := range allClients {
 		for _, peer := range allClients {
@@ -64,7 +67,7 @@ func TestResolveMagicDNS(t *testing.T) {
 				for _, ip := range ips {
 					assert.Contains(ct, result, ip.String(), "IP %s should be found in DNS resolution result from %s to %s", ip.String(), client.Hostname(), peer.Hostname())
 				}
-			}, 30*time.Second, 2*time.Second)
+			}, integrationutil.StatusReadyTimeout, 2*time.Second)
 		}
 	}
 }
@@ -78,26 +81,22 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 	}
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	const erPath = "/tmp/extra_records.json"
 
-	extraRecords := []tailcfg.DNSRecord{
-		{
-			Name:  "test.myvpn.example.com",
-			Type:  "A",
-			Value: "6.6.6.6",
-		},
-	}
-	b, _ := json.Marshal(extraRecords)
+	extraRecords := make([]tailcfg.DNSRecord, 0, 2)
+	extraRecords = append(extraRecords, tailcfg.DNSRecord{
+		Name:  "test.myvpn.example.com",
+		Type:  "A",
+		Value: "6.6.6.6",
+	})
+	b, _ := json.Marshal(extraRecords) //nolint:errchkjson
 
 	err = scenario.CreateHeadscaleEnv([]tsic.Option{
-		tsic.WithDockerEntrypoint([]string{
-			"/bin/sh",
-			"-c",
-			"/bin/sleep 3 ; apk add python3 curl bind-tools ; update-ca-certificates ; tailscaled --tun=tsdev",
-		}),
+		tsic.WithPackages("python3", "curl", "bind-tools"),
 	},
 		hsic.WithTestName("extrarecords"),
 		hsic.WithConfigEnv(map[string]string{
@@ -106,35 +105,33 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 			"HEADSCALE_DNS_EXTRA_RECORDS_PATH": erPath,
 		}),
 		hsic.WithFileInContainer(erPath, b),
-		hsic.WithEmbeddedDERPServerOnly(),
-		hsic.WithTLS(),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	// assertClientsState(t, allClients)
 
 	// Poor mans cache
 	_, err = scenario.ListTailscaleClientsFQDNs()
-	assertNoErrListFQDN(t, err)
+	requireNoErrListFQDN(t, err)
 
 	_, err = scenario.ListTailscaleClientsIPs()
-	assertNoErrListClientIPs(t, err)
+	requireNoErrListClientIPs(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "test.myvpn.example.com"}, "6.6.6.6")
 	}
 
 	hs, err := scenario.Headscale()
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	// Write the file directly into place from the docker API.
-	b0, _ := json.Marshal([]tailcfg.DNSRecord{
+	b0, _ := json.Marshal([]tailcfg.DNSRecord{ //nolint:errchkjson
 		{
 			Name:  "docker.myvpn.example.com",
 			Type:  "A",
@@ -143,7 +140,7 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 	})
 
 	err = hs.WriteFile(erPath, b0)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "docker.myvpn.example.com"}, "2.2.2.2")
@@ -156,12 +153,12 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 		Type:  "A",
 		Value: "7.7.7.7",
 	})
-	b2, _ := json.Marshal(extraRecords)
+	b2, _ := json.Marshal(extraRecords) //nolint:errchkjson
 
 	err = hs.WriteFile(erPath+"2", b2)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 	_, err = hs.Execute([]string{"mv", erPath + "2", erPath})
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "test.myvpn.example.com"}, "6.6.6.6")
@@ -170,7 +167,7 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 
 	// Write a new file and copy it to the path to ensure the reload
 	// works when a file is copied into place.
-	b3, _ := json.Marshal([]tailcfg.DNSRecord{
+	b3, _ := json.Marshal([]tailcfg.DNSRecord{ //nolint:errchkjson
 		{
 			Name:  "copy.myvpn.example.com",
 			Type:  "A",
@@ -179,16 +176,16 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 	})
 
 	err = hs.WriteFile(erPath+"3", b3)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 	_, err = hs.Execute([]string{"cp", erPath + "3", erPath})
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "copy.myvpn.example.com"}, "8.8.8.8")
 	}
 
 	// Write in place to ensure pipe like behaviour works
-	b4, _ := json.Marshal([]tailcfg.DNSRecord{
+	b4, _ := json.Marshal([]tailcfg.DNSRecord{ //nolint:errchkjson
 		{
 			Name:  "docker.myvpn.example.com",
 			Type:  "A",
@@ -197,7 +194,7 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 	})
 	command := []string{"echo", fmt.Sprintf("'%s'", string(b4)), ">", erPath}
 	_, err = hs.Execute([]string{"bash", "-c", strings.Join(command, " ")})
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "docker.myvpn.example.com"}, "9.9.9.9")
@@ -205,7 +202,7 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 
 	// Delete the file and create a new one to ensure it is picked up again.
 	_, err = hs.Execute([]string{"rm", erPath})
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	// The same paths should still be available as it is not cleared on delete.
 	assert.EventuallyWithT(t, func(ct *assert.CollectT) {
@@ -214,12 +211,12 @@ func TestResolveMagicDNSExtraRecordsPath(t *testing.T) {
 			assert.NoError(ct, err)
 			assert.Contains(ct, result, "9.9.9.9")
 		}
-	}, 10*time.Second, 1*time.Second)
+	}, integrationutil.ScaledTimeout(10*time.Second), 1*time.Second)
 
 	// Write a new file, the backoff mechanism should make the filewatcher pick it up
 	// again.
 	err = hs.WriteFile(erPath, b3)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	for _, client := range allClients {
 		assertCommandOutputContains(t, client, []string{"dig", "copy.myvpn.example.com"}, "8.8.8.8")

integration/dockertestutil/auth.go

@@ -0,0 +1,171 @@
+package dockertestutil
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/cenkalti/backoff/v5"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const dockerHubServer = "https://index.docker.io/v1/"
+
+type CredentialSource string
+
+const (
+	CredentialSourceEnv       CredentialSource = "env"
+	CredentialSourceConfig    CredentialSource = "config"
+	CredentialSourceAnonymous CredentialSource = "anonymous"
+)
+
+// Credentials resolves Docker Hub credentials from
+// DOCKERHUB_USERNAME/DOCKERHUB_TOKEN, then ~/.docker/config.json, then
+// anonymous. The Docker Go SDKs do not read config.json on their own.
+func Credentials() (string, string, CredentialSource) {
+	if u := os.Getenv("DOCKERHUB_USERNAME"); u != "" {
+		return u, os.Getenv("DOCKERHUB_TOKEN"), CredentialSourceEnv
+	}
+
+	user, pass, ok := credentialsFromConfig()
+	if ok {
+		return user, pass, CredentialSourceConfig
+	}
+
+	return "", "", CredentialSourceAnonymous
+}
+
+// AuthConfiguration returns Docker Hub auth for the dockertest pool.
+func AuthConfiguration() docker.AuthConfiguration {
+	u, p, _ := Credentials()
+
+	return docker.AuthConfiguration{
+		Username:      u,
+		Password:      p,
+		ServerAddress: dockerHubServer,
+	}
+}
+
+// RegistryAuth returns base64-encoded credentials for the modern
+// Docker SDK's image.PullOptions{RegistryAuth: ...}, or "" when none.
+func RegistryAuth() (string, error) {
+	u, p, _ := Credentials()
+	if u == "" && p == "" {
+		return "", nil
+	}
+
+	auth := struct {
+		Username string `json:"username"`
+		Password string `json:"password"`
+	}{Username: u, Password: p}
+
+	b, err := json.Marshal(auth) //nolint:gosec // G117: password field holds the Docker Hub token, intentional
+	if err != nil {
+		return "", fmt.Errorf("marshalling docker auth: %w", err)
+	}
+
+	return base64.URLEncoding.EncodeToString(b), nil
+}
+
+// PullWithAuth ensures imageRef is local, pulling with auth and
+// retrying transient errors when it is not.
+func PullWithAuth(pool *dockertest.Pool, imageRef string) error {
+	if img, _ := pool.Client.InspectImage(imageRef); img != nil {
+		return nil
+	}
+
+	repo, tag := splitImageRef(imageRef)
+	auth := AuthConfiguration()
+
+	_, err := backoff.Retry(
+		context.Background(),
+		func() (struct{}, error) {
+			err := pool.Client.PullImage(docker.PullImageOptions{
+				Repository: repo,
+				Tag:        tag,
+			}, auth)
+			if err == nil {
+				return struct{}{}, nil
+			}
+
+			if isPermanentPullError(err) {
+				return struct{}{}, backoff.Permanent(err)
+			}
+
+			return struct{}{}, fmt.Errorf("pulling %s: %w", imageRef, err)
+		},
+		backoff.WithBackOff(backoff.NewExponentialBackOff()),
+		backoff.WithMaxElapsedTime(60*time.Second),
+	)
+	if err != nil {
+		return fmt.Errorf("pulling %s with auth (source=%s): %w", imageRef, AuthConfiguration().ServerAddress, err)
+	}
+
+	return nil
+}
+
+func splitImageRef(ref string) (string, string) {
+	if i := strings.LastIndex(ref, ":"); i >= 0 {
+		return ref[:i], ref[i+1:]
+	}
+
+	return ref, "latest"
+}
+
+func isPermanentPullError(err error) bool {
+	msg := strings.ToLower(err.Error())
+
+	return strings.Contains(msg, "manifest unknown") ||
+		strings.Contains(msg, "manifest not found") ||
+		strings.Contains(msg, "repository does not exist") ||
+		strings.Contains(msg, "name unknown") ||
+		strings.Contains(msg, "no such image")
+}
+
+// credentialsFromConfig reads the Hub entry from ~/.docker/config.json.
+// Credential helpers (osxkeychain etc.) are not supported; use env vars.
+func credentialsFromConfig() (string, string, bool) {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return "", "", false
+	}
+
+	raw, err := os.ReadFile(filepath.Join(home, ".docker", "config.json"))
+	if err != nil {
+		return "", "", false
+	}
+
+	var cfg struct {
+		Auths map[string]struct {
+			Auth string `json:"auth"`
+		} `json:"auths"`
+	}
+
+	err = json.Unmarshal(raw, &cfg)
+	if err != nil {
+		return "", "", false
+	}
+
+	entry, found := cfg.Auths[dockerHubServer]
+	if !found || entry.Auth == "" {
+		return "", "", false
+	}
+
+	decoded, err := base64.StdEncoding.DecodeString(entry.Auth)
+	if err != nil {
+		return "", "", false
+	}
+
+	parts := strings.SplitN(string(decoded), ":", 2)
+	if len(parts) != 2 {
+		return "", "", false
+	}
+
+	return parts[0], parts[1], true
+}

integration/dockertestutil/build.go

@@ -0,0 +1,25 @@
+package dockertestutil
+
+import (
+	"context"
+	"os/exec"
+	"time"
+)
+
+// RunDockerBuildForDiagnostics runs docker build manually to get detailed error output.
+// This is used when a docker build fails to provide more detailed diagnostic information
+// than what dockertest typically provides.
+//
+// Returns the build output regardless of success/failure, and an error if the build failed.
+func RunDockerBuildForDiagnostics(contextDir, dockerfile string) (string, error) {
+	// Use a context with timeout to prevent hanging builds
+	const buildTimeout = 10 * time.Minute
+
+	ctx, cancel := context.WithTimeout(context.Background(), buildTimeout)
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, "docker", "build", "--progress=plain", "--no-cache", "-f", dockerfile, contextDir)
+	output, err := cmd.CombinedOutput()
+
+	return string(output), err
+}

integration/dockertestutil/config.go

@@ -22,9 +22,9 @@ func GetIntegrationRunID() string {
 	return os.Getenv("HEADSCALE_INTEGRATION_RUN_ID")
 }
 
-// DockerAddIntegrationLabels adds integration test labels to Docker RunOptions.
+// DockerAddIntegrationLabels adds integration test labels to Docker [dockertest.RunOptions].
 // This allows the hi tool to identify containers belonging to specific test runs.
-// This function should be called before passing RunOptions to dockertest functions.
+// This function should be called before passing [dockertest.RunOptions] to dockertest functions.
 func DockerAddIntegrationLabels(opts *dockertest.RunOptions, testType string) {
 	runID := GetIntegrationRunID()
 	if runID == "" {
@@ -34,6 +34,7 @@ func DockerAddIntegrationLabels(opts *dockertest.RunOptions, testType string) {
 	if opts.Labels == nil {
 		opts.Labels = make(map[string]string)
 	}
+
 	opts.Labels["hi.run-id"] = runID
 	opts.Labels["hi.test-type"] = testType
 }

integration/dockertestutil/execute.go

@@ -7,10 +7,20 @@ import (
 	"sync"
 	"time"
 
+	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/ory/dockertest/v3"
 )
 
-const dockerExecuteTimeout = time.Second * 10
+// defaultExecuteTimeout returns the timeout for docker exec commands.
+// On CI runners, docker exec latency is higher due to resource
+// contention, so the timeout is doubled.
+func defaultExecuteTimeout() time.Duration {
+	if util.IsCI() {
+		return 20 * time.Second
+	}
+
+	return 10 * time.Second
+}
 
 var (
 	ErrDockertestCommandFailed  = errors.New("dockertest command failed")
@@ -30,7 +40,7 @@ func ExecuteCommandTimeout(timeout time.Duration) ExecuteCommandOption {
 	})
 }
 
-// buffer is a goroutine safe bytes.buffer.
+// buffer is a goroutine safe [bytes.Buffer].
 type buffer struct {
 	store bytes.Buffer
 	mutex sync.Mutex
@@ -38,9 +48,10 @@ type buffer struct {
 
 // Write appends the contents of p to the buffer, growing the buffer as needed. It returns
 // the number of bytes written.
-func (b *buffer) Write(p []byte) (n int, err error) {
+func (b *buffer) Write(p []byte) (int, error) {
 	b.mutex.Lock()
 	defer b.mutex.Unlock()
+
 	return b.store.Write(p)
 }
 
@@ -49,6 +60,7 @@ func (b *buffer) Write(p []byte) (n int, err error) {
 func (b *buffer) String() string {
 	b.mutex.Lock()
 	defer b.mutex.Unlock()
+
 	return b.store.String()
 }
 
@@ -62,11 +74,12 @@ func ExecuteCommand(
 	stderr := buffer{}
 
 	execConfig := ExecuteCommandConfig{
-		timeout: dockerExecuteTimeout,
+		timeout: defaultExecuteTimeout(),
 	}
 
 	for _, opt := range options {
-		if err := opt(&execConfig); err != nil {
+		err := opt(&execConfig)
+		if err != nil {
 			return "", "", fmt.Errorf("execute-command/options: %w", err)
 		}
 	}
@@ -105,7 +118,6 @@ func ExecuteCommand(
 			// log.Println("Command: ", cmd)
 			// log.Println("stdout: ", stdout.String())
 			// log.Println("stderr: ", stderr.String())
-
 			return stdout.String(), stderr.String(), fmt.Errorf("command failed, stderr: %s: %w", stderr.String(), ErrDockertestCommandFailed)
 		}
 

integration/dockertestutil/logs.go

@@ -47,6 +47,7 @@ func SaveLog(
 	}
 
 	var stdout, stderr bytes.Buffer
+
 	err = WriteLog(pool, resource, &stdout, &stderr)
 	if err != nil {
 		return "", "", err
@@ -55,6 +56,7 @@ func SaveLog(
 	log.Printf("Saving logs for %s to %s\n", resource.Container.Name, basePath)
 
 	stdoutPath := path.Join(basePath, resource.Container.Name+".stdout.log")
+
 	err = os.WriteFile(
 		stdoutPath,
 		stdout.Bytes(),
@@ -65,6 +67,7 @@ func SaveLog(
 	}
 
 	stderrPath := path.Join(basePath, resource.Container.Name+".stderr.log")
+
 	err = os.WriteFile(
 		stderrPath,
 		stderr.Bytes(),

integration/dockertestutil/network.go

@@ -1,25 +1,74 @@
 package dockertestutil
 
 import (
+	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"log"
 	"net"
+	"strings"
+	"time"
 
+	"github.com/cenkalti/backoff/v5"
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/ory/dockertest/v3"
 	"github.com/ory/dockertest/v3/docker"
 )
 
-var ErrContainerNotFound = errors.New("container not found")
+var (
+	ErrContainerNotFound = errors.New("container not found")
+	ErrConditionTimeout  = errors.New("condition not met within timeout")
+)
+
+// retryDockerOp absorbs eventual-consistency races in libnetwork endpoint cleanup.
+// Pulls its backoff bounds from retry.go so every helper that drives a
+// docker control-plane call uses the same budget.
+func retryDockerOp(ctx context.Context, op func() error) error {
+	bo := backoff.NewExponentialBackOff()
+	bo.InitialInterval = DockerOpInitialInterval
+	bo.MaxInterval = DockerOpMaxInterval
+
+	_, err := backoff.Retry(ctx, func() (struct{}, error) {
+		err := op()
+		if err != nil {
+			return struct{}{}, err
+		}
+
+		return struct{}{}, nil
+	}, backoff.WithBackOff(bo), backoff.WithMaxElapsedTime(DockerOpMaxElapsedTime))
+
+	return err
+}
 
 func GetFirstOrCreateNetwork(pool *dockertest.Pool, name string) (*dockertest.Network, error) {
+	return GetFirstOrCreateNetworkWithSubnet(pool, name, "")
+}
+
+// GetFirstOrCreateNetworkWithSubnet creates a Docker network with an optional
+// custom subnet. When subnet is empty, Docker auto-assigns from its default
+// pool. Use RFC 5737 TEST-NET ranges (e.g. "198.51.100.0/24") for networks
+// that need to be reachable through Tailscale exit nodes, since Tailscale's
+// shrinkDefaultRoute strips RFC1918 ranges from exit node forwarding filters.
+func GetFirstOrCreateNetworkWithSubnet(pool *dockertest.Pool, name, subnet string) (*dockertest.Network, error) {
 	networks, err := pool.NetworksByName(name)
 	if err != nil {
 		return nil, fmt.Errorf("looking up network names: %w", err)
 	}
+
 	if len(networks) == 0 {
-		if _, err := pool.CreateNetwork(name); err == nil {
+		var opts []func(*docker.CreateNetworkOptions)
+		if subnet != "" {
+			opts = append(opts, func(config *docker.CreateNetworkOptions) {
+				config.IPAM = &docker.IPAMOptions{
+					Config: []docker.IPAMConfig{
+						{Subnet: subnet},
+					},
+				}
+			})
+		}
+
+		if _, err := pool.CreateNetwork(name, opts...); err == nil { //nolint:noinlineerr // intentional inline check
 			// Create does not give us an updated version of the resource, so we need to
 			// get it again.
 			networks, err := pool.NetworksByName(name)
@@ -51,13 +100,6 @@ func AddContainerToNetwork(
 		return err
 	}
 
-	err = pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
-		Container: containers[0].ID,
-	})
-	if err != nil {
-		return err
-	}
-
 	// TODO(kradalby): This doesn't work reliably, but calling the exact same functions
 	// seem to work fine...
 	// if container, ok := pool.ContainerByName("/" + testContainer); ok {
@@ -67,9 +109,276 @@ func AddContainerToNetwork(
 	// 	}
 	// }
 
+	return retryDockerOp(context.Background(), func() error {
+		return pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
+			Container: containers[0].ID,
+		})
+	})
+}
+
+// DisconnectContainerFromNetwork detaches the container at the docker
+// daemon level (cable-pull semantics) and waits for libnetwork to drop
+// the endpoint before returning — re-attaching during the
+// reprogramming window otherwise fails with "network is unreachable".
+func DisconnectContainerFromNetwork(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+) error {
+	containerID, err := lookupContainerID(pool, testContainer)
+	if err != nil {
+		return err
+	}
+
+	err = retryDockerOp(context.Background(), func() error {
+		return pool.Client.DisconnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
+			Container: containerID,
+		})
+	})
+	if err != nil {
+		return err
+	}
+
+	err = waitNetworkContainerAbsent(pool, network, testContainer, DockerOpMaxElapsedTime)
+	if err != nil {
+		return err
+	}
+
+	// libnetwork drops the endpoint from its model before the kernel
+	// netns has flushed the matching route. Re-attach with the sticky
+	// IP otherwise fails with "conflicts with existing route".
+	return waitContainerRouteAbsent(pool, containerID, network, DockerOpMaxElapsedTime)
+}
+
+// ReconnectContainerToNetwork is the inverse of
+// [DisconnectContainerFromNetwork] — re-attaches the container to the
+// network so traffic can flow again.
+func ReconnectContainerToNetwork(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+) error {
+	containerID, err := lookupContainerID(pool, testContainer)
+	if err != nil {
+		return err
+	}
+
+	err = retryDockerOp(context.Background(), func() error {
+		connectErr := pool.Client.ConnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{
+			Container: containerID,
+		})
+		if connectErr != nil && isStaleRouteConflict(connectErr) {
+			// Defensive cleanup: a route survived the netns flush
+			// despite the wait above. Drop subnet routes that point
+			// at the disconnected interface so libnetwork can
+			// reprogram the sticky IP, then let the retry budget
+			// try the ConnectNetwork call again.
+			removeContainerSubnetRoutes(pool, containerID, network)
+		}
+
+		return connectErr
+	})
+	if err != nil {
+		return err
+	}
+
+	return waitNetworkContainerPresent(pool, network, testContainer, DockerOpMaxElapsedTime)
+}
+
+// lookupContainerID resolves a container name to its docker ID.
+func lookupContainerID(pool *dockertest.Pool, testContainer string) (string, error) {
+	containers, err := pool.Client.ListContainers(docker.ListContainersOptions{
+		All:     true,
+		Filters: map[string][]string{"name": {testContainer}},
+	})
+	if err != nil {
+		return "", err
+	}
+
+	if len(containers) == 0 {
+		return "", fmt.Errorf("%w: %s", ErrContainerNotFound, testContainer)
+	}
+
+	return containers[0].ID, nil
+}
+
+// DisconnectAndReconnect calls Disconnect followed by Reconnect; both
+// primitives drive their own libnetwork settle waits.
+func DisconnectAndReconnect(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+) error {
+	err := DisconnectContainerFromNetwork(pool, network, testContainer)
+	if err != nil {
+		return fmt.Errorf("disconnecting %s from %s: %w", testContainer, network.Network.Name, err)
+	}
+
+	err = ReconnectContainerToNetwork(pool, network, testContainer)
+	if err != nil {
+		return fmt.Errorf("reconnecting %s to %s: %w", testContainer, network.Network.Name, err)
+	}
+
 	return nil
 }
 
+func waitNetworkContainerAbsent(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+	timeout time.Duration,
+) error {
+	return pollUntil(timeout, func() (bool, error) {
+		net, err := pool.Client.NetworkInfo(network.Network.ID)
+		if err != nil {
+			return false, fmt.Errorf("inspecting network %s: %w", network.Network.Name, err)
+		}
+
+		for _, c := range net.Containers {
+			if c.Name == testContainer || c.Name == "/"+testContainer {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}
+
+func waitNetworkContainerPresent(
+	pool *dockertest.Pool,
+	network *dockertest.Network,
+	testContainer string,
+	timeout time.Duration,
+) error {
+	return pollUntil(timeout, func() (bool, error) {
+		net, err := pool.Client.NetworkInfo(network.Network.ID)
+		if err != nil {
+			return false, fmt.Errorf("inspecting network %s: %w", network.Network.Name, err)
+		}
+
+		for _, c := range net.Containers {
+			if (c.Name == testContainer || c.Name == "/"+testContainer) && c.IPv4Address != "" {
+				return true, nil
+			}
+		}
+
+		return false, nil
+	})
+}
+
+// waitContainerRouteAbsent polls the container's routing table until no
+// route remains for the network's IPAM subnet. libnetwork's docker-side
+// endpoint teardown is asynchronous from the kernel netns flush, and a
+// surviving route blocks a subsequent reconnect at sticky-IP assignment
+// with "conflicts with existing route".
+func waitContainerRouteAbsent(pool *dockertest.Pool, containerID string, network *dockertest.Network, timeout time.Duration) error {
+	subnets := networkSubnets(network)
+	if len(subnets) == 0 {
+		return nil
+	}
+
+	return pollUntil(timeout, func() (bool, error) {
+		stdout, err := execStdout(pool, containerID, []string{"ip", "-4", "route", "show"})
+		if err != nil {
+			return false, fmt.Errorf("inspecting routes in %s: %w", containerID, err)
+		}
+
+		for _, subnet := range subnets {
+			if strings.Contains(stdout, subnet+" ") || strings.HasSuffix(strings.TrimSpace(stdout), subnet) {
+				return false, nil
+			}
+		}
+
+		return true, nil
+	})
+}
+
+// removeContainerSubnetRoutes drops residue subnet routes in the
+// container's netns — the leftover that libnetwork's async endpoint
+// teardown can leave behind.
+func removeContainerSubnetRoutes(pool *dockertest.Pool, containerID string, network *dockertest.Network) {
+	for _, subnet := range networkSubnets(network) {
+		_, err := execStdout(pool, containerID, []string{"ip", "-4", "route", "del", subnet})
+		if err != nil {
+			log.Printf("removing stale route %s in %s: %v", subnet, containerID, err)
+		}
+	}
+}
+
+// isStaleRouteConflict matches the libnetwork 500 raised when a
+// surviving subnet route blocks sticky-IP reprogramming on reconnect.
+func isStaleRouteConflict(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	return strings.Contains(err.Error(), "conflicts with existing route")
+}
+
+// networkSubnets returns the IPAM-configured subnets for a docker
+// network. Empty when IPAM is left to docker defaults.
+func networkSubnets(network *dockertest.Network) []string {
+	out := make([]string, 0, len(network.Network.IPAM.Config))
+	for _, cfg := range network.Network.IPAM.Config {
+		if cfg.Subnet != "" {
+			out = append(out, cfg.Subnet)
+		}
+	}
+
+	return out
+}
+
+// execStdout runs a one-shot command in containerID and returns stdout.
+func execStdout(pool *dockertest.Pool, containerID string, cmd []string) (string, error) {
+	exec, err := pool.Client.CreateExec(docker.CreateExecOptions{
+		Container:    containerID,
+		Cmd:          cmd,
+		AttachStdout: true,
+		AttachStderr: true,
+	})
+	if err != nil {
+		return "", fmt.Errorf("create exec: %w", err)
+	}
+
+	var stdout, stderr bytes.Buffer
+
+	err = pool.Client.StartExec(exec.ID, docker.StartExecOptions{
+		OutputStream: &stdout,
+		ErrorStream:  &stderr,
+	})
+	if err != nil {
+		return stdout.String(), fmt.Errorf("start exec: %w", err)
+	}
+
+	return stdout.String(), nil
+}
+
+// pollUntil ticks every DockerOpInitialInterval until check returns
+// done=true or timeout elapses. A non-nil check error aborts the loop.
+func pollUntil(timeout time.Duration, check func() (done bool, err error)) error {
+	deadline := time.Now().Add(timeout)
+
+	ticker := time.NewTicker(DockerOpInitialInterval)
+	defer ticker.Stop()
+
+	for {
+		done, err := check()
+		if err != nil {
+			return err
+		}
+
+		if done {
+			return nil
+		}
+
+		if time.Now().After(deadline) {
+			return fmt.Errorf("%w: %s", ErrConditionTimeout, timeout)
+		}
+
+		<-ticker.C
+	}
+}
+
 // RandomFreeHostPort asks the kernel for a free open port that is ready to use.
 // (from https://github.com/phayes/freeport)
 func RandomFreeHostPort() (int, error) {
@@ -90,6 +399,7 @@ func RandomFreeHostPort() (int, error) {
 // CleanUnreferencedNetworks removes networks that are not referenced by any containers.
 func CleanUnreferencedNetworks(pool *dockertest.Pool) error {
 	filter := "name=hs-"
+
 	networks, err := pool.NetworksByName(filter)
 	if err != nil {
 		return fmt.Errorf("getting networks by filter %q: %w", filter, err)
@@ -108,6 +418,8 @@ func CleanUnreferencedNetworks(pool *dockertest.Pool) error {
 }
 
 // CleanImagesInCI removes images if running in CI.
+// It only removes dangling (untagged) images to avoid forcing rebuilds.
+// Tagged images (golang:*, tailscale/tailscale:*, etc.) are automatically preserved.
 func CleanImagesInCI(pool *dockertest.Pool) error {
 	if !util.IsCI() {
 		log.Println("Skipping image cleanup outside of CI")
@@ -119,9 +431,27 @@ func CleanImagesInCI(pool *dockertest.Pool) error {
 		return fmt.Errorf("getting images: %w", err)
 	}
 
+	removedCount := 0
+
 	for _, image := range images {
-		log.Printf("removing image: %s, %v", image.ID, image.RepoTags)
-		_ = pool.Client.RemoveImage(image.ID)
+		// Only remove dangling (untagged) images to avoid forcing rebuilds
+		// Dangling images have no RepoTags or only have "<none>:<none>"
+		if len(image.RepoTags) == 0 || (len(image.RepoTags) == 1 && image.RepoTags[0] == "<none>:<none>") {
+			log.Printf("Removing dangling image: %s", image.ID[:12])
+
+			err := pool.Client.RemoveImage(image.ID)
+			if err != nil {
+				log.Printf("Warning: failed to remove image %s: %v", image.ID[:12], err)
+			} else {
+				removedCount++
+			}
+		}
+	}
+
+	if removedCount > 0 {
+		log.Printf("Removed %d dangling images in CI", removedCount)
+	} else {
+		log.Println("No dangling images to remove in CI")
 	}
 
 	return nil

integration/dockertestutil/network_test.go

@@ -0,0 +1,56 @@
+package dockertestutil
+
+import (
+	"context"
+	"errors"
+	"sync/atomic"
+	"testing"
+	"time"
+)
+
+var (
+	errTransientEndpointExists = errors.New("endpoint with name foo already exists in network bar")
+	errPermanent               = errors.New("permanent error")
+)
+
+func TestRetryDockerOp_RecoversFromTransient(t *testing.T) {
+	var attempts atomic.Int32
+
+	op := func() error {
+		if attempts.Add(1) < 3 {
+			return errTransientEndpointExists
+		}
+
+		return nil
+	}
+
+	err := retryDockerOp(context.Background(), op)
+	if err != nil {
+		t.Fatalf("retryDockerOp should recover from 2 transient errors, got: %v", err)
+	}
+
+	if got := attempts.Load(); got != 3 {
+		t.Fatalf("expected 3 attempts, got %d", got)
+	}
+}
+
+func TestRetryDockerOp_RespectsContextCancellation(t *testing.T) {
+	op := func() error {
+		return errPermanent
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	err := retryDockerOp(ctx, op)
+	elapsed := time.Since(start)
+
+	if err == nil {
+		t.Fatal("retryDockerOp should fail when op always errors")
+	}
+
+	if elapsed > 5*time.Second {
+		t.Fatalf("retryDockerOp should honour ctx deadline (~200ms), took %s", elapsed)
+	}
+}

integration/dockertestutil/retry.go

@@ -0,0 +1,12 @@
+package dockertestutil
+
+import "time"
+
+// Docker control-plane retry policy. MaxElapsedTime sits above the
+// worst observed libnetwork bridge reprogramming time (~60 s on
+// contended GHA runners).
+const (
+	DockerOpInitialInterval = 1 * time.Second
+	DockerOpMaxInterval     = 10 * time.Second
+	DockerOpMaxElapsedTime  = 90 * time.Second
+)

integration/dsic/dsic.go

@@ -40,6 +40,7 @@ type DERPServerInContainer struct {
 	stunPort            int
 	derpPort            int
 	caCerts             [][]byte
+	tlsCACert           []byte
 	tlsCert             []byte
 	tlsKey              []byte
 	withExtraHosts      []string
@@ -75,7 +76,7 @@ func WithOrCreateNetwork(network *dockertest.Network) Option {
 			dsic.hostname+"-network",
 		)
 		if err != nil {
-			log.Fatalf("failed to create network: %s", err)
+			log.Fatalf("creating network: %s", err)
 		}
 
 		dsic.networks = append(dsic.networks, network)
@@ -103,7 +104,39 @@ func WithExtraHosts(hosts []string) Option {
 	}
 }
 
-// New returns a new TailscaleInContainer instance.
+// buildEntrypoint builds the container entrypoint command based on configuration.
+// It constructs proper wait conditions instead of fixed sleeps:
+// 1. Wait for network to be ready
+// 2. Wait for TLS cert to be written (always written after container start)
+// 3. Wait for CA certs if configured
+// 4. Update CA certificates
+// 5. Run derper with provided arguments.
+func (dsic *DERPServerInContainer) buildEntrypoint(derperArgs string) []string {
+	var commands []string
+
+	// Wait for network to be ready
+	commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
+
+	// Wait for TLS cert to be written (always written after container start)
+	commands = append(commands,
+		fmt.Sprintf("while [ ! -f %s/%s.crt ]; do sleep 0.1; done", DERPerCertRoot, dsic.hostname))
+
+	// If CA certs are configured, wait for them to be written
+	if len(dsic.caCerts) > 0 {
+		commands = append(commands,
+			fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot))
+	}
+
+	// Update CA certificates
+	commands = append(commands, "update-ca-certificates")
+
+	// Run derper
+	commands = append(commands, "derper "+derperArgs)
+
+	return []string{"/bin/sh", "-c", strings.Join(commands, " ; ")}
+}
+
+// New returns a new [tsic.TailscaleInContainer] instance.
 func New(
 	pool *dockertest.Pool,
 	version string,
@@ -115,22 +148,40 @@ func New(
 		return nil, err
 	}
 
-	hostname := fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash)
-	tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname)
+	// Include run ID in hostname for easier identification of which test run owns this container
+	runID := dockertestutil.GetIntegrationRunID()
+
+	var hostname string
+
+	if runID != "" {
+		// Use last 6 chars of run ID (the random hash part) for brevity
+		runIDShort := runID[len(runID)-6:]
+		hostname = fmt.Sprintf("derp-%s-%s-%s", runIDShort, strings.ReplaceAll(version, ".", "-"), hash)
+	} else {
+		hostname = fmt.Sprintf("derp-%s-%s", strings.ReplaceAll(version, ".", "-"), hash)
+	}
+
+	tlsCACert, tlsCert, tlsKey, err := integrationutil.CreateCertificate(hostname)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create certificates for headscale test: %w", err)
+		return nil, fmt.Errorf("creating certificates for derp test: %w", err)
 	}
+
 	dsic := &DERPServerInContainer{
-		version:  version,
-		hostname: hostname,
-		pool:     pool,
-		networks: networks,
-		tlsCert:  tlsCert,
-		tlsKey:   tlsKey,
-		stunPort: 3478, //nolint
-		derpPort: 443,  //nolint
+		version:   version,
+		hostname:  hostname,
+		pool:      pool,
+		networks:  networks,
+		tlsCACert: tlsCACert,
+		tlsCert:   tlsCert,
+		tlsKey:    tlsKey,
+		stunPort:  3478, //nolint
+		derpPort:  443,  //nolint
 	}
 
+	// Install the CA cert so the DERP server trusts its own certificate
+	// and any headscale CA certs passed via [WithCACert].
+	dsic.caCerts = append(dsic.caCerts, tlsCACert)
+
 	for _, opt := range opts {
 		opt(dsic)
 	}
@@ -142,6 +193,7 @@ func New(
 	fmt.Fprintf(&cmdArgs, " --a=:%d", dsic.derpPort)
 	fmt.Fprintf(&cmdArgs, " --stun=true")
 	fmt.Fprintf(&cmdArgs, " --stun-port=%d", dsic.stunPort)
+
 	if dsic.withVerifyClientURL != "" {
 		fmt.Fprintf(&cmdArgs, " --verify-client-url=%s", dsic.withVerifyClientURL)
 	}
@@ -150,8 +202,7 @@ func New(
 		Name:       hostname,
 		Networks:   dsic.networks,
 		ExtraHosts: dsic.withExtraHosts,
-		// we currently need to give us some time to inject the certificate further down.
-		Entrypoint: []string{"/bin/sh", "-c", "/bin/sleep 3 ; update-ca-certificates ; derper " + cmdArgs.String()},
+		Entrypoint: dsic.buildEntrypoint(cmdArgs.String()),
 		ExposedPorts: []string{
 			"80/tcp",
 			fmt.Sprintf("%d/tcp", dsic.derpPort),
@@ -172,11 +223,13 @@ func New(
 	}
 
 	var container *dockertest.Resource
+
 	buildOptions := &dockertest.BuildOptions{
 		Dockerfile: "Dockerfile.derper",
 		ContextDir: dockerContextPath,
 		BuildArgs:  []docker.BuildArg{},
 	}
+
 	switch version {
 	case "head":
 		buildOptions.BuildArgs = append(buildOptions.BuildArgs, docker.BuildArg{
@@ -201,12 +254,13 @@ func New(
 	)
 	if err != nil {
 		return nil, fmt.Errorf(
-			"%s could not start tailscale DERPer container (version: %s): %w",
+			"%s starting tailscale DERPer container (version: %s): %w",
 			hostname,
 			version,
 			err,
 		)
 	}
+
 	log.Printf("Created %s container\n", hostname)
 
 	dsic.container = container
@@ -214,19 +268,21 @@ func New(
 	for i, cert := range dsic.caCerts {
 		err = dsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
 		if err != nil {
-			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+			return nil, fmt.Errorf("writing TLS certificate to container: %w", err)
 		}
 	}
+
 	if len(dsic.tlsCert) != 0 {
 		err = dsic.WriteFile(fmt.Sprintf("%s/%s.crt", DERPerCertRoot, dsic.hostname), dsic.tlsCert)
 		if err != nil {
-			return nil, fmt.Errorf("failed to write TLS certificate to container: %w", err)
+			return nil, fmt.Errorf("writing TLS certificate to container: %w", err)
 		}
 	}
+
 	if len(dsic.tlsKey) != 0 {
 		err = dsic.WriteFile(fmt.Sprintf("%s/%s.key", DERPerCertRoot, dsic.hostname), dsic.tlsKey)
 		if err != nil {
-			return nil, fmt.Errorf("failed to write TLS key to container: %w", err)
+			return nil, fmt.Errorf("writing TLS key to container: %w", err)
 		}
 	}
 
@@ -238,18 +294,19 @@ func (t *DERPServerInContainer) Shutdown() error {
 	err := t.SaveLog("/tmp/control")
 	if err != nil {
 		log.Printf(
-			"Failed to save log from %s: %s",
+			"saving log from %s: %s",
 			t.hostname,
-			fmt.Errorf("failed to save log: %w", err),
+			fmt.Errorf("saving log: %w", err),
 		)
 	}
 
 	return t.pool.Purge(t.container)
 }
 
-// GetCert returns the TLS certificate of the DERPer instance.
+// GetCert returns the CA certificate that clients should trust to
+// verify this DERP server's TLS certificate.
 func (t *DERPServerInContainer) GetCert() []byte {
-	return t.tlsCert
+	return t.tlsCACert
 }
 
 // Hostname returns the hostname of the DERPer instance.
@@ -262,7 +319,7 @@ func (t *DERPServerInContainer) Version() string {
 	return t.version
 }
 
-// ID returns the Docker container ID of the DERPServerInContainer
+// ID returns the Docker container ID of the [DERPServerInContainer]
 // instance.
 func (t *DERPServerInContainer) ID() string {
 	return t.container.Container.ID
@@ -294,7 +351,7 @@ func (t *DERPServerInContainer) WaitForRunning() error {
 	return t.pool.Retry(func() error {
 		resp, err := client.Get(url) //nolint
 		if err != nil {
-			return fmt.Errorf("headscale is not ready: %w", err)
+			return fmt.Errorf("DERPer is not ready: %w", err)
 		}
 
 		if resp.StatusCode != http.StatusOK {

integration/embedded_derp_test.go

@@ -5,8 +5,10 @@ import (
 	"time"
 
 	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/integrationutil"
 	"github.com/juanfont/headscale/integration/tsic"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 )
@@ -20,16 +22,16 @@ func TestDERPServerScenario(t *testing.T) {
 	spec := ScenarioSpec{
 		NodesPerUser: 1,
 		Users:        []string{"user1", "user2", "user3"},
-		Networks: map[string][]string{
-			"usernet1": {"user1"},
-			"usernet2": {"user2"},
-			"usernet3": {"user3"},
+		Networks: map[string]NetworkSpec{
+			"usernet1": {Users: []string{"user1"}},
+			"usernet2": {Users: []string{"user2"}},
+			"usernet3": {Users: []string{"user3"}},
 		},
 	}
 
-	derpServerScenario(t, spec, false, func(scenario *Scenario) {
+	derpServerScenario(t, spec, "derp-tcp", false, func(scenario *Scenario) {
 		allClients, err := scenario.ListTailscaleClients()
-		assertNoErrListClients(t, err)
+		requireNoErrListClients(t, err)
 		t.Logf("checking %d clients for websocket connections", len(allClients))
 
 		for _, client := range allClients {
@@ -43,7 +45,7 @@ func TestDERPServerScenario(t *testing.T) {
 		}
 
 		hsServer, err := scenario.Headscale()
-		assertNoErrGetHeadscale(t, err)
+		requireNoErrGetHeadscale(t, err)
 
 		derpRegion := tailcfg.DERPRegion{
 			RegionCode: "test-derpverify",
@@ -70,16 +72,16 @@ func TestDERPServerWebsocketScenario(t *testing.T) {
 	spec := ScenarioSpec{
 		NodesPerUser: 1,
 		Users:        []string{"user1", "user2", "user3"},
-		Networks: map[string][]string{
-			"usernet1": {"user1"},
-			"usernet2": {"user2"},
-			"usernet3": {"user3"},
+		Networks: map[string]NetworkSpec{
+			"usernet1": {Users: []string{"user1"}},
+			"usernet2": {Users: []string{"user2"}},
+			"usernet3": {Users: []string{"user3"}},
 		},
 	}
 
-	derpServerScenario(t, spec, true, func(scenario *Scenario) {
+	derpServerScenario(t, spec, "derp-ws", true, func(scenario *Scenario) {
 		allClients, err := scenario.ListTailscaleClients()
-		assertNoErrListClients(t, err)
+		requireNoErrListClients(t, err)
 		t.Logf("checking %d clients for websocket connections", len(allClients))
 
 		for _, client := range allClients {
@@ -102,13 +104,14 @@ func TestDERPServerWebsocketScenario(t *testing.T) {
 func derpServerScenario(
 	t *testing.T,
 	spec ScenarioSpec,
+	testName string,
 	websocket bool,
 	furtherAssertions ...func(*Scenario),
 ) {
 	IntegrationSkip(t)
 
 	scenario, err := NewScenario(spec)
-	assertNoErr(t, err)
+	require.NoError(t, err)
 
 	defer scenario.ShutdownAssertNoPanics(t)
 
@@ -116,11 +119,11 @@ func derpServerScenario(
 		[]tsic.Option{
 			tsic.WithWebsocketDERP(websocket),
 		},
-		hsic.WithTestName("derpserver"),
+		hsic.WithTestName(testName),
+		// Expose STUN port for DERP NAT traversal.
 		hsic.WithExtraPorts([]string{"3478/udp"}),
-		hsic.WithEmbeddedDERPServerOnly(),
+		// DERP clients expect the server on the standard HTTPS port.
 		hsic.WithPort(443),
-		hsic.WithTLS(),
 		hsic.WithConfigEnv(map[string]string{
 			"HEADSCALE_DERP_AUTO_UPDATE_ENABLED":   "true",
 			"HEADSCALE_DERP_UPDATE_FREQUENCY":      "10s",
@@ -128,16 +131,16 @@ func derpServerScenario(
 			"HEADSCALE_DERP_SERVER_VERIFY_CLIENTS": "true",
 		}),
 	)
-	assertNoErrHeadscaleEnv(t, err)
+	requireNoErrHeadscaleEnv(t, err)
 
 	allClients, err := scenario.ListTailscaleClients()
-	assertNoErrListClients(t, err)
+	requireNoErrListClients(t, err)
 
 	err = scenario.WaitForTailscaleSync()
-	assertNoErrSync(t, err)
+	requireNoErrSync(t, err)
 
 	allHostnames, err := scenario.ListTailscaleClientsFQDNs()
-	assertNoErrListFQDN(t, err)
+	requireNoErrListFQDN(t, err)
 
 	for _, client := range allClients {
 		assert.EventuallyWithT(t, func(ct *assert.CollectT) {
@@ -150,7 +153,7 @@ func derpServerScenario(
 				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.",
 					"Client %s should be connected to Headscale Embedded DERP", client.Hostname())
 			}
-		}, 30*time.Second, 2*time.Second)
+		}, integrationutil.StatusReadyTimeout, 2*time.Second)
 	}
 
 	success := pingDerpAllHelper(t, allClients, allHostnames)
@@ -171,13 +174,14 @@ func derpServerScenario(
 				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.",
 					"Client %s should be connected to Headscale Embedded DERP after first run", client.Hostname())
 			}
-		}, 30*time.Second, 2*time.Second)
+		}, integrationutil.StatusReadyTimeout, 2*time.Second)
 	}
 
 	t.Logf("Run 1: %d successful pings out of %d", success, len(allClients)*len(allHostnames))
 
 	// Let the DERP updater run a couple of times to ensure it does not
-	// break the DERPMap.
+	// break the [tailcfg.DERPMap]. The updater runs on a 10s interval by default.
+	//nolint:forbidigo // Intentional delay: must wait for DERP updater to run multiple times (interval-based)
 	time.Sleep(30 * time.Second)
 
 	success = pingDerpAllHelper(t, allClients, allHostnames)
@@ -196,7 +200,7 @@ func derpServerScenario(
 				assert.NotContains(ct, health, "could not connect to the 'Headscale Embedded DERP' relay server.",
 					"Client %s should be connected to Headscale Embedded DERP after second run", client.Hostname())
 			}
-		}, 30*time.Second, 2*time.Second)
+		}, integrationutil.StatusReadyTimeout, 2*time.Second)
 	}
 
 	t.Logf("Run2: %d successful pings out of %d", success, len(allClients)*len(allHostnames))

integration/grant_cap_test.go

@@ -0,0 +1,994 @@
+package integration
+
+import (
+	"fmt"
+	"net/netip"
+	"strings"
+	"testing"
+	"time"
+
+	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
+	"github.com/juanfont/headscale/hscontrol/types"
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"tailscale.com/tailcfg"
+	"tailscale.com/wgengine/filter"
+)
+
+// hasCapMatchInPacketFilter checks if any [filter.Match] entry in the packet
+// filter contains a [filter.CapMatch] with the given capability name.
+func hasCapMatchInPacketFilter(pf []filter.Match, peerCap tailcfg.PeerCapability) bool {
+	for _, m := range pf {
+		for _, cm := range m.Caps {
+			if cm.Cap == peerCap {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// hasCapMatchForIP checks if any [filter.CapMatch] with the given capability
+// has a Dst prefix that contains the given IP. This validates that
+// the cap is directed at the correct node, not just present.
+func hasCapMatchForIP(pf []filter.Match, peerCap tailcfg.PeerCapability, ip netip.Addr) bool {
+	for _, m := range pf {
+		for _, cm := range m.Caps {
+			if cm.Cap == peerCap && cm.Dst.Contains(ip) {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
+// parsePeerRelay parses a PeerRelay string of the form "ip:port:vni:N"
+// and returns the [netip.AddrPort] and VNI. Returns zero values on parse failure.
+func parsePeerRelay(pr string) (netip.AddrPort, string, bool) {
+	// Format: "172.18.0.4:58738:vni:1"
+	// Split into: host part "172.18.0.4:58738" and vni part "vni:1"
+	addrStr, vni, ok := strings.Cut(pr, ":vni:")
+	if !ok {
+		return netip.AddrPort{}, "", false
+	}
+
+	ap, err := netip.ParseAddrPort(addrStr)
+	if err != nil {
+		return netip.AddrPort{}, "", false
+	}
+
+	return ap, vni, true
+}
+
+// TestGrantCapRelay validates the full peer relay lifecycle:
+//  1. No direct connection between isolated clients
+//  2. Cap grants compile correctly (relay + relay-target in packet filters)
+//     with strict directionality and negative checks
+//  3. Peer relay is used instead of DERP (PeerRelay non-empty, valid format)
+//  4. Relay goes down -> fallback to DERP (PeerRelay empty, Relay non-empty,
+//     DERP ping works)
+//  5. Relay comes back up -> peer relay resumes (PeerRelay non-empty again)
+func TestGrantCapRelay(t *testing.T) {
+	IntegrationSkip(t)
+
+	assertTimeout := 120 * time.Second
+
+	spec := ScenarioSpec{
+		NodesPerUser: 0,
+		Users:        []string{"relay", "clienta", "clientb"},
+		Networks: map[string]NetworkSpec{
+			"usernet1": {Users: []string{"clienta"}},
+			"usernet2": {Users: []string{"clientb"}},
+			"usernet3": {Users: []string{"relay"}},
+		},
+		Versions: []string{"head"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoErrorf(t, err, "failed to create scenario: %s", err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	pol := &policyv2.Policy{
+		TagOwners: policyv2.TagOwners{
+			policyv2.Tag("tag:relay"):    policyv2.Owners{usernameOwner("relay@")},
+			policyv2.Tag("tag:client-a"): policyv2.Owners{usernameOwner("clienta@")},
+			policyv2.Tag("tag:client-b"): policyv2.Owners{usernameOwner("clientb@")},
+		},
+		Grants: []policyv2.Grant{
+			// Grant 1: Basic IP connectivity between all tagged nodes.
+			{
+				Sources: policyv2.Aliases{
+					tagp("tag:relay"), tagp("tag:client-a"), tagp("tag:client-b"),
+				},
+				Destinations: policyv2.Aliases{
+					tagp("tag:relay"), tagp("tag:client-a"), tagp("tag:client-b"),
+				},
+				InternetProtocols: []policyv2.ProtocolPort{
+					{Protocol: "*", Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}},
+				},
+			},
+			// Grant 2: Relay cap - clients can use relay node for UDP relaying.
+			// This generates cap/relay on the relay's filter and cap/relay-target
+			// (companion) on the clients' filters.
+			{
+				Sources:      policyv2.Aliases{tagp("tag:client-a"), tagp("tag:client-b")},
+				Destinations: policyv2.Aliases{tagp("tag:relay")},
+				App: tailcfg.PeerCapMap{
+					tailcfg.PeerCapabilityRelay: {tailcfg.RawMessage("{}")},
+				},
+			},
+		},
+	}
+
+	headscale, err := scenario.Headscale(
+		hsic.WithTestName("grant-cap-relay"),
+		hsic.WithACLPolicy(pol),
+		hsic.WithPolicyMode(types.PolicyModeDB),
+	)
+	requireNoErrGetHeadscale(t, err)
+
+	usernet1, err := scenario.Network("usernet1")
+	require.NoError(t, err)
+	usernet2, err := scenario.Network("usernet2")
+	require.NoError(t, err)
+	usernet3, err := scenario.Network("usernet3")
+	require.NoError(t, err)
+
+	// Create users on headscale server.
+	_, err = scenario.CreateUser("relay")
+	require.NoError(t, err)
+	_, err = scenario.CreateUser("clienta")
+	require.NoError(t, err)
+	_, err = scenario.CreateUser("clientb")
+	require.NoError(t, err)
+
+	userMap, err := headscale.MapUsers()
+	require.NoError(t, err)
+
+	// --- Create Relay R on usernet3, dual-homed to usernet1+usernet2 ---
+	relayR, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet3),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = relayR.Shutdown() }()
+
+	pakRelay, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["relay"].GetId(), false, false, []string{"tag:relay"},
+	)
+	require.NoError(t, err)
+	err = relayR.Login(headscale.GetEndpoint(), pakRelay.GetKey())
+	require.NoError(t, err)
+	err = relayR.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// Dual-home after registration to avoid duplicate node key generation
+	// from Docker network interface changes during tailscaled startup.
+	err = relayR.ConnectToNetwork(usernet1)
+	require.NoError(t, err)
+	err = relayR.ConnectToNetwork(usernet2)
+	require.NoError(t, err)
+
+	// Enable the relay server on the relay node. Without this, the
+	// relayserver extension loads but RelayServerPort is nil and the
+	// server never starts listening for allocation requests.
+	// Port 0 = random unused port.
+	_, _, err = relayR.Execute([]string{
+		"tailscale", "set", "--relay-server-port=0",
+	})
+	require.NoError(t, err)
+
+	// --- Create Client A on usernet1 only ---
+	clientA, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet1),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = clientA.Shutdown() }()
+
+	pakClientA, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["clienta"].GetId(), false, false, []string{"tag:client-a"},
+	)
+	require.NoError(t, err)
+	err = clientA.Login(headscale.GetEndpoint(), pakClientA.GetKey())
+	require.NoError(t, err)
+	err = clientA.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// --- Create Client B on usernet2 only ---
+	clientB, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet2),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = clientB.Shutdown() }()
+
+	pakClientB, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["clientb"].GetId(), false, false, []string{"tag:client-b"},
+	)
+	require.NoError(t, err)
+	err = clientB.Login(headscale.GetEndpoint(), pakClientB.GetKey())
+	require.NoError(t, err)
+	err = clientB.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// ===== Phase 1: Validate isolation and peer visibility =====
+	t.Log("Phase 1: Validate network isolation and peer visibility")
+
+	allNodes := []TailscaleClient{relayR, clientA, clientB}
+	for _, node := range allNodes {
+		err = node.WaitForPeers(len(allNodes)-1, 60*time.Second, 1*time.Second)
+		require.NoErrorf(t, err, "node %s failed to see all peers", node.Hostname())
+	}
+
+	// Restart all nodes to ensure fresh wireguard config. When nodes
+	// register sequentially, early peers may arrive without DERP info
+	// and get permanently skipped in wireguard config.
+	for _, node := range allNodes {
+		require.NoError(t, node.Restart())
+		require.NoError(t, node.WaitForRunning(30*time.Second))
+	}
+
+	for _, node := range allNodes {
+		err = node.WaitForPeers(len(allNodes)-1, 60*time.Second, 1*time.Second)
+		require.NoErrorf(t, err, "node %s failed to see all peers after restart", node.Hostname())
+	}
+
+	// Capture keys and IPs for assertions.
+	clientBKey := clientB.MustStatus().Self.PublicKey
+	clientAKey := clientA.MustStatus().Self.PublicKey
+	relayIPv4 := relayR.MustIPv4()
+	clientAIPv4 := clientA.MustIPv4()
+	clientBIPv4 := clientB.MustIPv4()
+
+	// Verify no direct path between A and B.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := clientA.Status()
+		assert.NoError(c, err)
+
+		peerB := status.Peer[clientBKey]
+		assert.NotNil(c, peerB, "A should see B as a peer")
+
+		if peerB != nil {
+			assert.Empty(c, peerB.CurAddr, "A->B should have no direct path")
+		}
+	}, assertTimeout, 500*time.Millisecond, "A should have no direct path to B")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := clientB.Status()
+		assert.NoError(c, err)
+
+		peerA := status.Peer[clientAKey]
+		assert.NotNil(c, peerA, "B should see A as a peer")
+
+		if peerA != nil {
+			assert.Empty(c, peerA.CurAddr, "B->A should have no direct path")
+		}
+	}, assertTimeout, 500*time.Millisecond, "B should have no direct path to A")
+
+	// ===== Phase 2: Validate cap grants in packet filters =====
+	t.Log("Phase 2: Validate cap grants in packet filters")
+
+	// --- Positive checks: correct caps on correct nodes ---
+
+	// Relay R should have cap/relay targeting the relay's own IP.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := relayR.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityRelay, relayIPv4),
+			"Relay R should have cap/relay with Dst matching relay's IP %s", relayIPv4)
+	}, assertTimeout, 500*time.Millisecond, "R should have cap/relay targeting its own IP")
+
+	// Client A should have cap/relay-target targeting client A's IP.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := clientA.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityRelayTarget, clientAIPv4),
+			"Client A should have cap/relay-target with Dst matching A's IP %s", clientAIPv4)
+	}, assertTimeout, 500*time.Millisecond, "A should have cap/relay-target targeting its own IP")
+
+	// Client B should have cap/relay-target targeting client B's IP.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := clientB.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityRelayTarget, clientBIPv4),
+			"Client B should have cap/relay-target with Dst matching B's IP %s", clientBIPv4)
+	}, assertTimeout, 500*time.Millisecond, "B should have cap/relay-target targeting its own IP")
+
+	// --- Negative checks: wrong caps must NOT be present ---
+
+	// Relay R should NOT have cap/relay-target (it's a relay server, not a target).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := relayR.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityRelayTarget),
+			"Relay R should NOT have cap/relay-target")
+	}, 10*time.Second, 500*time.Millisecond, "R should not have cap/relay-target")
+
+	// Client A should NOT have cap/relay (it's a client, not a relay server).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := clientA.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityRelay),
+			"Client A should NOT have cap/relay")
+	}, 10*time.Second, 500*time.Millisecond, "A should not have cap/relay")
+
+	// Client B should NOT have cap/relay (it's a client, not a relay server).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := clientB.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityRelay),
+			"Client B should NOT have cap/relay")
+	}, 10*time.Second, 500*time.Millisecond, "B should not have cap/relay")
+
+	// ===== Phase 3: Validate peer relay active (not DERP) =====
+	t.Log("Phase 3: Validate peer relay active (not DERP)")
+
+	// Verify PeerRelay is set with valid format and correct relay IPs.
+	// Relay endpoint allocation is triggered by traffic between peers,
+	// so we send pings in the check loop to initiate relay discovery.
+	var peerRelayAtoB, peerRelayBtoA string
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		// Fire a ping to trigger relay path discovery (ignore output).
+		clientA.Execute([]string{"tailscale", "ping", "--c=1", "--timeout=1s", clientBIPv4.String()}) //nolint:errcheck
+
+		status, err := clientA.Status()
+		assert.NoError(c, err)
+
+		peerB := status.Peer[clientBKey]
+		assert.NotNil(c, peerB, "A should see B as a peer")
+
+		if peerB != nil {
+			assert.NotEmpty(c, peerB.PeerRelay,
+				"A->B should use peer relay, not DERP")
+			assert.Empty(c, peerB.CurAddr,
+				"A->B should not have direct connection")
+
+			if peerB.PeerRelay != "" {
+				peerRelayAtoB = peerB.PeerRelay
+
+				// Validate PeerRelay format: ip:port:vni:N
+				ap, vni, ok := parsePeerRelay(peerB.PeerRelay)
+				assert.True(c, ok,
+					"PeerRelay %q should be parseable as ip:port:vni:N", peerB.PeerRelay)
+
+				if ok {
+					assert.NotZero(c, ap.Port(),
+						"PeerRelay port should be non-zero")
+					assert.NotEmpty(c, vni,
+						"PeerRelay VNI should be non-empty")
+				}
+			}
+
+			t.Logf("Phase 3 - A->B: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerB.PeerRelay, peerB.Relay, peerB.CurAddr, peerB.Active)
+		}
+	}, assertTimeout, 2*time.Second, "A should show peer relay to B")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		clientB.Execute([]string{"tailscale", "ping", "--c=1", "--timeout=1s", clientAIPv4.String()}) //nolint:errcheck
+
+		status, err := clientB.Status()
+		assert.NoError(c, err)
+
+		peerA := status.Peer[clientAKey]
+		assert.NotNil(c, peerA, "B should see A as a peer")
+
+		if peerA != nil {
+			assert.NotEmpty(c, peerA.PeerRelay,
+				"B->A should use peer relay, not DERP")
+
+			if peerA.PeerRelay != "" {
+				peerRelayBtoA = peerA.PeerRelay
+
+				ap, vni, ok := parsePeerRelay(peerA.PeerRelay)
+				assert.True(c, ok,
+					"PeerRelay %q should be parseable as ip:port:vni:N", peerA.PeerRelay)
+
+				if ok {
+					assert.NotZero(c, ap.Port(),
+						"PeerRelay port should be non-zero")
+					assert.NotEmpty(c, vni,
+						"PeerRelay VNI should be non-empty")
+				}
+			}
+
+			t.Logf("Phase 3 - B->A: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerA.PeerRelay, peerA.Relay, peerA.CurAddr, peerA.Active)
+		}
+	}, assertTimeout, 2*time.Second, "B should show peer relay to A")
+
+	// Cross-validate: both directions should use the same VNI
+	// (same relay allocation) but different IPs (dual-homed relay).
+	if peerRelayAtoB != "" && peerRelayBtoA != "" {
+		apA, vniA, okA := parsePeerRelay(peerRelayAtoB)
+
+		apB, vniB, okB := parsePeerRelay(peerRelayBtoA)
+		if okA && okB {
+			assert.Equal(t, vniA, vniB,
+				"A->B and B->A should share the same VNI (same relay allocation)")
+			assert.Equal(t, apA.Port(), apB.Port(),
+				"A->B and B->A should use the same relay port")
+			assert.NotEqual(t, apA.Addr(), apB.Addr(),
+				"A->B and B->A relay IPs should differ (dual-homed relay)")
+		}
+	}
+
+	// ===== Phase 4: Bring relay down -> DERP fallback =====
+	t.Log("Phase 4: Bring relay down, expect DERP fallback")
+
+	require.NoError(t, relayR.Down())
+
+	// Verify PeerRelay is gone and DERP is used.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := clientA.Status()
+		assert.NoError(c, err)
+
+		peerB := status.Peer[clientBKey]
+		assert.NotNil(c, peerB, "A should still see B as a peer")
+
+		if peerB != nil {
+			assert.Empty(c, peerB.PeerRelay,
+				"A->B peer relay should be gone")
+			assert.NotEmpty(c, peerB.Relay,
+				"A->B should fall back to DERP")
+			t.Logf("Phase 4 - A->B: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerB.PeerRelay, peerB.Relay, peerB.CurAddr, peerB.Active)
+		}
+	}, assertTimeout, 500*time.Millisecond, "A should fall back to DERP for B")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := clientB.Status()
+		assert.NoError(c, err)
+
+		peerA := status.Peer[clientAKey]
+		assert.NotNil(c, peerA, "B should still see A as a peer")
+
+		if peerA != nil {
+			assert.Empty(c, peerA.PeerRelay,
+				"B->A peer relay should be gone")
+			t.Logf("Phase 4 - B->A: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerA.PeerRelay, peerA.Relay, peerA.CurAddr, peerA.Active)
+		}
+	}, assertTimeout, 500*time.Millisecond, "B should fall back to DERP for A")
+
+	// Verify data plane works via DERP after relay is down.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		err := clientA.Ping(
+			clientBIPv4.String(),
+			tsic.WithPingUntilDirect(false),
+			tsic.WithPingTimeout(2*time.Second),
+			tsic.WithPingCount(1),
+		)
+		assert.NoError(c, err)
+	}, assertTimeout, 1*time.Second, "A should reach B via DERP after relay down")
+
+	// ===== Phase 5: Bring relay back up -> peer relay resumes =====
+	t.Log("Phase 5: Bring relay back up, expect peer relay to resume")
+
+	require.NoError(t, relayR.Up())
+
+	err = relayR.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// Verify peer relay resumes. Ping to trigger relay re-discovery.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		clientA.Execute([]string{"tailscale", "ping", "--c=1", "--timeout=1s", clientBIPv4.String()}) //nolint:errcheck
+
+		status, err := clientA.Status()
+		assert.NoError(c, err)
+
+		peerB := status.Peer[clientBKey]
+		assert.NotNil(c, peerB, "A should see B as a peer")
+
+		if peerB != nil {
+			assert.NotEmpty(c, peerB.PeerRelay,
+				"A->B peer relay should resume after R comes back")
+			t.Logf("Phase 5 - A->B: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerB.PeerRelay, peerB.Relay, peerB.CurAddr, peerB.Active)
+		}
+	}, assertTimeout, 2*time.Second, "A should resume peer relay to B")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		clientB.Execute([]string{"tailscale", "ping", "--c=1", "--timeout=1s", clientAIPv4.String()}) //nolint:errcheck
+
+		status, err := clientB.Status()
+		assert.NoError(c, err)
+
+		peerA := status.Peer[clientAKey]
+		assert.NotNil(c, peerA, "B should see A as a peer")
+
+		if peerA != nil {
+			assert.NotEmpty(c, peerA.PeerRelay,
+				"B->A peer relay should resume after R comes back")
+			t.Logf("Phase 5 - B->A: PeerRelay=%q Relay=%q CurAddr=%q Active=%v",
+				peerA.PeerRelay, peerA.Relay, peerA.CurAddr, peerA.Active)
+		}
+	}, assertTimeout, 2*time.Second, "B should resume peer relay to A")
+}
+
+// driveURL constructs a Taildrive WebDAV URL via the local proxy.
+func driveURL(domain, sharerName, path string) string {
+	return fmt.Sprintf(
+		"http://100.100.100.100:8080/%s/%s/%s",
+		domain, sharerName, path,
+	)
+}
+
+// TestGrantCapDrive validates Taildrive (cap/drive) grant-based access control:
+//  1. Node attributes (drive:share, drive:access) are set in all nodes' CapMap
+//  2. Cap grants compile correctly (cap/drive + cap/drive-sharer in packet filters)
+//  3. RW client can read, write, and delete files on the sharer
+//  4. RO client can read but NOT write or delete files on the sharer
+//  5. No-access node (no cap/drive grant) cannot read or write files
+func TestGrantCapDrive(t *testing.T) {
+	IntegrationSkip(t)
+
+	assertTimeout := 120 * time.Second
+
+	spec := ScenarioSpec{
+		NodesPerUser: 0,
+		Users:        []string{"sharer", "rwclient", "roclient", "noaccess"},
+		Networks: map[string]NetworkSpec{
+			"usernet1": {Users: []string{"sharer", "rwclient", "roclient", "noaccess"}},
+		},
+		Versions: []string{"head"},
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoErrorf(t, err, "failed to create scenario: %s", err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	pol := &policyv2.Policy{
+		TagOwners: policyv2.TagOwners{
+			policyv2.Tag("tag:sharer"):    policyv2.Owners{usernameOwner("sharer@")},
+			policyv2.Tag("tag:rw-client"): policyv2.Owners{usernameOwner("rwclient@")},
+			policyv2.Tag("tag:ro-client"): policyv2.Owners{usernameOwner("roclient@")},
+			policyv2.Tag("tag:no-access"): policyv2.Owners{usernameOwner("noaccess@")},
+		},
+		// Taildrive caps (drive:share / drive:access) are policy-driven
+		// per https://tailscale.com/docs/features/taildrive; no longer
+		// emitted as TailNode baseline. Stamp on every node so the
+		// SelfNode.CapMap assertions below remain meaningful.
+		NodeAttrs: []policyv2.NodeAttrGrant{
+			{
+				Targets: policyv2.Aliases{policyv2.Wildcard},
+				Attrs: []tailcfg.NodeCapability{
+					tailcfg.NodeAttrsTaildriveShare,
+					tailcfg.NodeAttrsTaildriveAccess,
+				},
+			},
+		},
+		Grants: []policyv2.Grant{
+			// Grant 1: IP connectivity between ALL nodes.
+			{
+				Sources: policyv2.Aliases{
+					tagp("tag:sharer"), tagp("tag:rw-client"),
+					tagp("tag:ro-client"), tagp("tag:no-access"),
+				},
+				Destinations: policyv2.Aliases{
+					tagp("tag:sharer"), tagp("tag:rw-client"),
+					tagp("tag:ro-client"), tagp("tag:no-access"),
+				},
+				InternetProtocols: []policyv2.ProtocolPort{
+					{Protocol: "*", Ports: []tailcfg.PortRange{tailcfg.PortRangeAny}},
+				},
+			},
+			// Grant 2: cap/drive RW - rw-client can read+write sharer's drives.
+			{
+				Sources:      policyv2.Aliases{tagp("tag:rw-client")},
+				Destinations: policyv2.Aliases{tagp("tag:sharer")},
+				App: tailcfg.PeerCapMap{
+					tailcfg.PeerCapabilityTaildrive: {
+						tailcfg.RawMessage(`{"shares":["*"],"access":"rw"}`),
+					},
+				},
+			},
+			// Grant 3: cap/drive RO - ro-client can only read sharer's drives.
+			{
+				Sources:      policyv2.Aliases{tagp("tag:ro-client")},
+				Destinations: policyv2.Aliases{tagp("tag:sharer")},
+				App: tailcfg.PeerCapMap{
+					tailcfg.PeerCapabilityTaildrive: {
+						tailcfg.RawMessage(`{"shares":["*"],"access":"ro"}`),
+					},
+				},
+			},
+			// NO cap/drive grant for tag:no-access (intentional).
+		},
+	}
+
+	headscale, err := scenario.Headscale(
+		hsic.WithTestName("grant-cap-drive"),
+		hsic.WithACLPolicy(pol),
+		hsic.WithPolicyMode(types.PolicyModeDB),
+	)
+	requireNoErrGetHeadscale(t, err)
+
+	usernet1, err := scenario.Network("usernet1")
+	require.NoError(t, err)
+
+	// Create users on headscale server.
+	_, err = scenario.CreateUser("sharer")
+	require.NoError(t, err)
+	_, err = scenario.CreateUser("rwclient")
+	require.NoError(t, err)
+	_, err = scenario.CreateUser("roclient")
+	require.NoError(t, err)
+	_, err = scenario.CreateUser("noaccess")
+	require.NoError(t, err)
+
+	userMap, err := headscale.MapUsers()
+	require.NoError(t, err)
+
+	// --- Create Sharer node ---
+	sharer, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet1),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = sharer.Shutdown() }()
+
+	pakSharer, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["sharer"].GetId(), false, false, []string{"tag:sharer"},
+	)
+	require.NoError(t, err)
+	err = sharer.Login(headscale.GetEndpoint(), pakSharer.GetKey())
+	require.NoError(t, err)
+	err = sharer.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// --- Create RW client node ---
+	rwClient, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet1),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = rwClient.Shutdown() }()
+
+	pakRW, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["rwclient"].GetId(), false, false, []string{"tag:rw-client"},
+	)
+	require.NoError(t, err)
+	err = rwClient.Login(headscale.GetEndpoint(), pakRW.GetKey())
+	require.NoError(t, err)
+	err = rwClient.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// --- Create RO client node ---
+	roClient, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet1),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = roClient.Shutdown() }()
+
+	pakRO, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["roclient"].GetId(), false, false, []string{"tag:ro-client"},
+	)
+	require.NoError(t, err)
+	err = roClient.Login(headscale.GetEndpoint(), pakRO.GetKey())
+	require.NoError(t, err)
+	err = roClient.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// --- Create No-access node ---
+	noAccess, err := scenario.CreateTailscaleNode("head",
+		tsic.WithNetwork(usernet1),
+	)
+	require.NoError(t, err)
+
+	defer func() { _, _, _ = noAccess.Shutdown() }()
+
+	pakNA, err := scenario.CreatePreAuthKeyWithTags(
+		userMap["noaccess"].GetId(), false, false, []string{"tag:no-access"},
+	)
+	require.NoError(t, err)
+	err = noAccess.Login(headscale.GetEndpoint(), pakNA.GetKey())
+	require.NoError(t, err)
+	err = noAccess.WaitForRunning(30 * time.Second)
+	require.NoError(t, err)
+
+	// ===== Phase 1: Wait for all peers =====
+	t.Log("Phase 1: Wait for all peers to be visible")
+
+	allNodes := []TailscaleClient{sharer, rwClient, roClient, noAccess}
+	for _, node := range allNodes {
+		err = node.WaitForPeers(len(allNodes)-1, 60*time.Second, 1*time.Second)
+		require.NoErrorf(t, err, "node %s failed to see all peers", node.Hostname())
+	}
+
+	sharerIPv4 := sharer.MustIPv4()
+
+	// ===== Phase 2: Validate node attributes (self CapMap) =====
+	t.Log("Phase 2: Validate Taildrive node attributes in CapMap")
+
+	for _, node := range allNodes {
+		assert.EventuallyWithT(t, func(c *assert.CollectT) {
+			nm, err := node.Netmap()
+			assert.NoError(c, err)
+
+			if nm == nil {
+				return
+			}
+
+			assert.True(c, nm.SelfNode.Valid(),
+				"%s: SelfNode should be valid", node.Hostname())
+
+			if nm.SelfNode.Valid() {
+				assert.True(c, nm.SelfNode.HasCap(tailcfg.NodeAttrsTaildriveShare),
+					"%s: should have drive:share cap", node.Hostname())
+				assert.True(c, nm.SelfNode.HasCap(tailcfg.NodeAttrsTaildriveAccess),
+					"%s: should have drive:access cap", node.Hostname())
+			}
+		}, assertTimeout, 500*time.Millisecond,
+			"all nodes should have Taildrive node attributes")
+	}
+
+	// ===== Phase 3: Validate cap grants in packet filters =====
+	t.Log("Phase 3: Validate cap/drive grants in packet filters")
+
+	// --- Positive checks ---
+
+	// Sharer should have cap/drive targeting its own IP (it's the drive destination).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := sharer.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityTaildrive, sharerIPv4),
+			"Sharer should have cap/drive with Dst matching sharer's IP %s", sharerIPv4)
+	}, assertTimeout, 500*time.Millisecond, "sharer should have cap/drive targeting its own IP")
+
+	// RW client should have cap/drive-sharer (companion) targeting rw-client's IP.
+	rwClientIPv4 := rwClient.MustIPv4()
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := rwClient.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityTaildriveSharer, rwClientIPv4),
+			"RW client should have cap/drive-sharer with Dst matching rw-client's IP %s", rwClientIPv4)
+	}, assertTimeout, 500*time.Millisecond, "rw-client should have cap/drive-sharer")
+
+	// RO client should have cap/drive-sharer (companion) targeting ro-client's IP.
+	roClientIPv4 := roClient.MustIPv4()
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := roClient.PacketFilter()
+		assert.NoError(c, err)
+		assert.True(c, hasCapMatchForIP(pf, tailcfg.PeerCapabilityTaildriveSharer, roClientIPv4),
+			"RO client should have cap/drive-sharer with Dst matching ro-client's IP %s", roClientIPv4)
+	}, assertTimeout, 500*time.Millisecond, "ro-client should have cap/drive-sharer")
+
+	// --- Negative checks ---
+
+	// No-access node should NOT have cap/drive or cap/drive-sharer.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := noAccess.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityTaildrive),
+			"no-access should NOT have cap/drive")
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityTaildriveSharer),
+			"no-access should NOT have cap/drive-sharer")
+	}, 10*time.Second, 500*time.Millisecond, "no-access should have no drive caps")
+
+	// Sharer should NOT have cap/drive-sharer (it's a destination, not a source).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := sharer.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityTaildriveSharer),
+			"sharer should NOT have cap/drive-sharer")
+	}, 10*time.Second, 500*time.Millisecond, "sharer should not have cap/drive-sharer")
+
+	// RW client should NOT have cap/drive (it's a source, not a destination).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := rwClient.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityTaildrive),
+			"rw-client should NOT have cap/drive")
+	}, 10*time.Second, 500*time.Millisecond, "rw-client should not have cap/drive")
+
+	// RO client should NOT have cap/drive (it's a source, not a destination).
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		pf, err := roClient.PacketFilter()
+		assert.NoError(c, err)
+		assert.False(c, hasCapMatchInPacketFilter(pf, tailcfg.PeerCapabilityTaildrive),
+			"ro-client should NOT have cap/drive")
+	}, 10*time.Second, 500*time.Millisecond, "ro-client should not have cap/drive")
+
+	// ===== Phase 4: Create share on sharer =====
+	t.Log("Phase 4: Create share on sharer node")
+
+	_, _, err = sharer.Execute([]string{"mkdir", "-p", "/tmp/testshare"})
+	require.NoError(t, err)
+	_, _, err = sharer.Execute([]string{
+		"sh", "-c", `echo "hello-taildrive" > /tmp/testshare/testfile.txt`,
+	})
+	require.NoError(t, err)
+	_, _, err = sharer.Execute([]string{
+		"tailscale", "drive", "share", "testshare", "/tmp/testshare",
+	})
+	require.NoError(t, err)
+
+	// Verify share is listed.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := sharer.Execute([]string{"tailscale", "drive", "list"})
+		assert.NoError(c, err)
+		assert.Contains(c, result, "testshare",
+			"sharer should list 'testshare' in drive list")
+	}, 10*time.Second, 500*time.Millisecond, "sharer should have testshare listed")
+
+	// Build the drive URL components from the sharer's FQDN.
+	fqdn := strings.TrimSuffix(sharer.MustFQDN(), ".")
+	parts := strings.SplitN(fqdn, ".", 2)
+	sharerName := parts[0]
+	domain := parts[1]
+
+	// ===== Phase 5: RW client - read file (positive) =====
+	t.Log("Phase 5: RW client reads file from sharer")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := rwClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			driveURL(domain, sharerName, "testshare/testfile.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Equal(c, "hello-taildrive", strings.TrimSpace(result),
+			"rw-client should read testfile.txt content")
+	}, 60*time.Second, 2*time.Second, "rw-client should read file from sharer")
+
+	// ===== Phase 6: RW client - write file (positive) =====
+	t.Log("Phase 6: RW client writes file to sharer")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := rwClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			"-X", "PUT", "--data-binary", "written-by-rw",
+			driveURL(domain, sharerName, "testshare/rw-wrote.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Contains(c, result, "20",
+			"rw-client PUT should return 2xx status")
+	}, 30*time.Second, 2*time.Second, "rw-client should write file to sharer")
+
+	// Verify the file exists on sharer.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		content, err := sharer.ReadFile("/tmp/testshare/rw-wrote.txt")
+		assert.NoError(c, err)
+		assert.Equal(c, "written-by-rw", strings.TrimSpace(string(content)))
+	}, 10*time.Second, 500*time.Millisecond, "rw-wrote.txt should exist on sharer")
+
+	// ===== Phase 7: RO client - read file (positive) =====
+	t.Log("Phase 7: RO client reads file from sharer")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := roClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			driveURL(domain, sharerName, "testshare/testfile.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Equal(c, "hello-taildrive", strings.TrimSpace(result),
+			"ro-client should read testfile.txt content")
+	}, 60*time.Second, 2*time.Second, "ro-client should read file from sharer")
+
+	// ===== Phase 8: RO client - write file (NEGATIVE - expect 403) =====
+	t.Log("Phase 8: RO client write attempt (should be denied)")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := roClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			"-X", "PUT", "--data-binary", "should-not-work",
+			driveURL(domain, sharerName, "testshare/ro-wrote.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Equal(c, "403", strings.TrimSpace(result),
+			"ro-client PUT should return 403 Forbidden")
+	}, 30*time.Second, 2*time.Second, "ro-client write should be 403 Forbidden")
+
+	// Verify file was NOT created on sharer.
+	_, err = sharer.ReadFile("/tmp/testshare/ro-wrote.txt")
+	require.Error(t, err, "ro-wrote.txt should not exist on sharer")
+
+	// ===== Phase 9: No-access node - read file (NEGATIVE) =====
+	t.Log("Phase 9: No-access node read attempt (should fail)")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := noAccess.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			driveURL(domain, sharerName, "testshare/testfile.txt"),
+		})
+		// Either error (connection refused) or non-200 status.
+		if err == nil {
+			assert.NotEqual(c, "200", strings.TrimSpace(result),
+				"no-access node should NOT get 200 from sharer's drive")
+		}
+	}, 30*time.Second, 2*time.Second, "no-access node should not read sharer's files")
+
+	// ===== Phase 10: No-access node - write file (NEGATIVE) =====
+	t.Log("Phase 10: No-access node write attempt (should fail)")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := noAccess.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			"-X", "PUT", "--data-binary", "should-not-work",
+			driveURL(domain, sharerName, "testshare/no-access-wrote.txt"),
+		})
+		if err == nil {
+			assert.NotEqual(c, "200", strings.TrimSpace(result),
+				"no-access node should not get 200 on PUT")
+			assert.NotEqual(c, "201", strings.TrimSpace(result),
+				"no-access node should not get 201 on PUT")
+		}
+	}, 30*time.Second, 2*time.Second, "no-access node should not write sharer's files")
+
+	// Verify file NOT created on sharer.
+	_, err = sharer.ReadFile("/tmp/testshare/no-access-wrote.txt")
+	require.Error(t, err, "no-access-wrote.txt should not exist on sharer")
+
+	// ===== Phase 11: RW client - list directory via PROPFIND (positive) =====
+	t.Log("Phase 11: RW client lists directory via PROPFIND")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := rwClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-X", "PROPFIND", "-H", "Depth: 1",
+			driveURL(domain, sharerName, "testshare/"),
+		})
+		assert.NoError(c, err)
+		assert.Contains(c, result, "testfile.txt",
+			"PROPFIND should list testfile.txt")
+		assert.Contains(c, result, "rw-wrote.txt",
+			"PROPFIND should list rw-wrote.txt")
+	}, 30*time.Second, 2*time.Second, "rw-client PROPFIND should list files")
+
+	// ===== Phase 12: RW client - delete file (positive) =====
+	t.Log("Phase 12: RW client deletes file from sharer")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := rwClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			"-X", "DELETE",
+			driveURL(domain, sharerName, "testshare/rw-wrote.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Contains(c, result, "20",
+			"rw-client DELETE should return 2xx status")
+	}, 30*time.Second, 2*time.Second, "rw-client should delete file from sharer")
+
+	// Verify deleted on sharer.
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		_, err := sharer.ReadFile("/tmp/testshare/rw-wrote.txt")
+		assert.Error(c, err, "rw-wrote.txt should be deleted from sharer")
+	}, 10*time.Second, 500*time.Millisecond, "rw-wrote.txt should be gone")
+
+	// ===== Phase 13: RO client - delete file (NEGATIVE - expect 403) =====
+	t.Log("Phase 13: RO client delete attempt (should be denied)")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, _, err := roClient.Execute([]string{
+			"curl", "-s", "--max-time", "5",
+			"-o", "/dev/null", "-w", "%{http_code}",
+			"-X", "DELETE",
+			driveURL(domain, sharerName, "testshare/testfile.txt"),
+		})
+		assert.NoError(c, err)
+		assert.Equal(c, "403", strings.TrimSpace(result),
+			"ro-client DELETE should return 403 Forbidden")
+	}, 30*time.Second, 2*time.Second, "ro-client delete should be 403 Forbidden")
+
+	// Verify file still exists on sharer.
+	content, err := sharer.ReadFile("/tmp/testshare/testfile.txt")
+	require.NoError(t, err)
+	assert.Equal(t, "hello-taildrive", strings.TrimSpace(string(content)),
+		"testfile.txt should still exist after RO delete attempt")
+}

integration/hsic/config.go

@@ -28,11 +28,24 @@ func DefaultConfigEnv() map[string]string {
 		"HEADSCALE_PRIVATE_KEY_PATH":                  "/tmp/private.key",
 		"HEADSCALE_NOISE_PRIVATE_KEY_PATH":            "/tmp/noise_private.key",
 		"HEADSCALE_METRICS_LISTEN_ADDR":               "0.0.0.0:9090",
-		"HEADSCALE_DERP_URLS":                         "https://controlplane.tailscale.com/derpmap/default",
-		"HEADSCALE_DERP_AUTO_UPDATE_ENABLED":          "false",
-		"HEADSCALE_DERP_UPDATE_FREQUENCY":             "1m",
 		"HEADSCALE_DEBUG_PORT":                        "40000",
 
+		// Embedded DERP is the default for test isolation.
+		// Tests should not depend on external DERP infrastructure.
+		// Use [WithPublicDERP] to opt out for tests that explicitly
+		// need public DERP relays.
+		"HEADSCALE_DERP_URLS":                    "",
+		"HEADSCALE_DERP_AUTO_UPDATE_ENABLED":     "false",
+		"HEADSCALE_DERP_UPDATE_FREQUENCY":        "1m",
+		"HEADSCALE_DERP_SERVER_ENABLED":          "true",
+		"HEADSCALE_DERP_SERVER_REGION_ID":        "999",
+		"HEADSCALE_DERP_SERVER_REGION_CODE":      binHeadscale,
+		"HEADSCALE_DERP_SERVER_REGION_NAME":      "Headscale Embedded DERP",
+		"HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR": "0.0.0.0:3478",
+		"HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH": "/tmp/derp.key",
+		"DERP_DEBUG_LOGS":                        "true",
+		"DERP_PROBER_DEBUG_LOGS":                 "true",
+
 		// a bunch of tests (ACL/Policy) rely on predictable IP alloc,
 		// so ensure the sequential alloc is used by default.
 		"HEADSCALE_PREFIXES_ALLOCATION": string(types.IPAllocationStrategySequential),

integration/integrationutil/timeouts.go

@@ -0,0 +1,40 @@
+package integrationutil
+
+import "time"
+
+// CI-scaled convergence budgets. ScaledTimeout doubles each on CI.
+var (
+	// HAConvergeTimeout: routes / ACL / policy propagation to reach
+	// every node and show up in status, traceroute, or curl.
+	HAConvergeTimeout = ScaledTimeout(60 * time.Second)
+
+	// HASlowConvergeTimeout: multi-step failover sequences that need
+	// at least one HA prober cycle plus a data-plane settle.
+	HASlowConvergeTimeout = ScaledTimeout(120 * time.Second)
+
+	// PolicyPropagationTimeout: post-SetPolicy filter rules and peer
+	// reachability to reflect the change. Sized for wgengine's
+	// rule-reload lag on contended CI runners (~2 min observed).
+	PolicyPropagationTimeout = ScaledTimeout(180 * time.Second)
+
+	// AuthFlowTimeout: OIDC / web-auth / preauth-key flows to reach
+	// Running.
+	AuthFlowTimeout = ScaledTimeout(30 * time.Second)
+
+	// StatusReadyTimeout: post-event read to reflect the event
+	// (created node visible in list, set tags visible on node).
+	StatusReadyTimeout = ScaledTimeout(30 * time.Second)
+)
+
+// Polling intervals for [assert.EventuallyWithT].
+const (
+	// FastPoll: in-process reads (HA state, route table snapshots).
+	FastPoll = 200 * time.Millisecond
+
+	// SlowPoll: cross-container reads (tailscale status, curl,
+	// headscale API) where each tick pays a docker exec round-trip.
+	SlowPoll = 500 * time.Millisecond
+
+	// PingPoll: gap between full ping-matrix sweeps.
+	PingPoll = 2 * time.Second
+)

integration/integrationutil/util.go

@@ -28,6 +28,7 @@ func PeerSyncTimeout() time.Duration {
 	if util.IsCI() {
 		return 120 * time.Second
 	}
+
 	return 60 * time.Second
 }
 
@@ -36,6 +37,18 @@ func PeerSyncRetryInterval() time.Duration {
 	return 100 * time.Millisecond
 }
 
+// ScaledTimeout returns the given timeout, scaled for CI environments
+// where resource contention causes slower state propagation.
+// Uses a 2x multiplier, consistent with PeerSyncTimeout (60s/120s)
+// and dockertestMaxWait (300s/600s).
+func ScaledTimeout(d time.Duration) time.Duration {
+	if util.IsCI() {
+		return d * 2
+	}
+
+	return d
+}
+
 func WriteFileToContainer(
 	pool *dockertest.Pool,
 	container *dockertest.Resource,
@@ -59,17 +72,17 @@ func WriteFileToContainer(
 
 	err := tarWriter.WriteHeader(header)
 	if err != nil {
-		return fmt.Errorf("failed write file header to tar: %w", err)
+		return fmt.Errorf("writing file header to tar: %w", err)
 	}
 
 	_, err = io.Copy(tarWriter, file)
 	if err != nil {
-		return fmt.Errorf("failed to copy file to tar: %w", err)
+		return fmt.Errorf("copying file to tar: %w", err)
 	}
 
 	err = tarWriter.Close()
 	if err != nil {
-		return fmt.Errorf("failed to close tar: %w", err)
+		return fmt.Errorf("closing tar: %w", err)
 	}
 
 	// Ensure the directory is present inside the container
@@ -79,7 +92,7 @@ func WriteFileToContainer(
 		[]string{},
 	)
 	if err != nil {
-		return fmt.Errorf("failed to ensure directory: %w", err)
+		return fmt.Errorf("ensuring directory: %w", err)
 	}
 
 	err = pool.Client.UploadToContainer(
@@ -119,7 +132,11 @@ func FetchPathFromContainer(
 }
 
 // nolint
-func CreateCertificate(hostname string) ([]byte, []byte, error) {
+// CreateCertificate generates a CA certificate and a server certificate
+// signed by that CA for the given hostname. It returns the CA certificate
+// PEM (for trust stores), server certificate PEM, and server private key
+// PEM.
+func CreateCertificate(hostname string) (caCertPEM, certPEM, keyPEM []byte, err error) {
 	// From:
 	// https://shaneutt.com/blog/golang-ca-and-signed-cert-go/
 
@@ -143,7 +160,27 @@ func CreateCertificate(hostname string) ([]byte, []byte, error) {
 
 	caPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
+	}
+
+	caBytes, err := x509.CreateCertificate(
+		rand.Reader,
+		ca,
+		ca,
+		&caPrivKey.PublicKey,
+		caPrivKey,
+	)
+	if err != nil {
+		return nil, nil, nil, err
+	}
+
+	caPEM := new(bytes.Buffer)
+	err = pem.Encode(caPEM, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: caBytes,
+	})
+	if err != nil {
+		return nil, nil, nil, err
 	}
 
 	cert := &x509.Certificate{
@@ -164,7 +201,7 @@ func CreateCertificate(hostname string) ([]byte, []byte, error) {
 
 	certPrivKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	certBytes, err := x509.CreateCertificate(
@@ -175,55 +212,55 @@ func CreateCertificate(hostname string) ([]byte, []byte, error) {
 		caPrivKey,
 	)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	certPEM := new(bytes.Buffer)
-
-	err = pem.Encode(certPEM, &pem.Block{
+	serverCertPEM := new(bytes.Buffer)
+	err = pem.Encode(serverCertPEM, &pem.Block{
 		Type:  "CERTIFICATE",
 		Bytes: certBytes,
 	})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
 	certPrivKeyPEM := new(bytes.Buffer)
-
 	err = pem.Encode(certPrivKeyPEM, &pem.Block{
 		Type:  "RSA PRIVATE KEY",
 		Bytes: x509.MarshalPKCS1PrivateKey(certPrivKey),
 	})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	return certPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
+	return caPEM.Bytes(), serverCertPEM.Bytes(), certPrivKeyPEM.Bytes(), nil
 }
 
 func BuildExpectedOnlineMap(all map[types.NodeID][]tailcfg.MapResponse) map[types.NodeID]map[types.NodeID]bool {
 	res := make(map[types.NodeID]map[types.NodeID]bool)
 	for nid, mrs := range all {
 		res[nid] = make(map[types.NodeID]bool)
+
 		for _, mr := range mrs {
 			for _, peer := range mr.Peers {
 				if peer.Online != nil {
-					res[nid][types.NodeID(peer.ID)] = *peer.Online
+					res[nid][types.NodeID(peer.ID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
 				}
 			}
 
 			for _, peer := range mr.PeersChanged {
 				if peer.Online != nil {
-					res[nid][types.NodeID(peer.ID)] = *peer.Online
+					res[nid][types.NodeID(peer.ID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
 				}
 			}
 
 			for _, peer := range mr.PeersChangedPatch {
 				if peer.Online != nil {
-					res[nid][types.NodeID(peer.NodeID)] = *peer.Online
+					res[nid][types.NodeID(peer.NodeID)] = *peer.Online //nolint:gosec // safe conversion for peer ID
 				}
 			}
 		}
 	}
+
 	return res
 }

integration/scenario_test.go

@@ -4,7 +4,9 @@ import (
 	"testing"
 
 	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/hsic"
 	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/stretchr/testify/require"
 )
 
 // This file is intended to "test the test framework", by proxy it will also test
@@ -34,11 +36,12 @@ func TestHeadscale(t *testing.T) {
 	user := "test-space"
 
 	scenario, err := NewScenario(ScenarioSpec{})
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	t.Run("start-headscale", func(t *testing.T) {
-		headscale, err := scenario.Headscale()
+		headscale, err := scenario.Headscale(hsic.WithTestName("scenariohs"))
 		if err != nil {
 			t.Fatalf("failed to create start headcale: %s", err)
 		}
@@ -82,11 +85,12 @@ func TestTailscaleNodesJoiningHeadcale(t *testing.T) {
 	count := 1
 
 	scenario, err := NewScenario(ScenarioSpec{})
-	assertNoErr(t, err)
+
+	require.NoError(t, err)
 	defer scenario.ShutdownAssertNoPanics(t)
 
 	t.Run("start-headscale", func(t *testing.T) {
-		headscale, err := scenario.Headscale()
+		headscale, err := scenario.Headscale(hsic.WithTestName("scenariojoin"))
 		if err != nil {
 			t.Fatalf("failed to create start headcale: %s", err)
 		}

integration/tailscale.go

@@ -10,10 +10,12 @@ import (
 	"github.com/juanfont/headscale/hscontrol/util"
 	"github.com/juanfont/headscale/integration/dockertestutil"
 	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/ory/dockertest/v3"
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/net/netcheck"
 	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
+	"tailscale.com/wgengine/filter"
 )
 
 // nolint
@@ -28,6 +30,7 @@ type TailscaleClient interface {
 	Login(loginServer, authKey string) error
 	LoginWithURL(loginServer string) (*url.URL, error)
 	Logout() error
+	Restart() error
 	Up() error
 	Down() error
 	IPs() ([]netip.Addr, error)
@@ -36,6 +39,7 @@ type TailscaleClient interface {
 	MustIPv4() netip.Addr
 	MustIPv6() netip.Addr
 	FQDN() (string, error)
+	MustFQDN() string
 	Status(...bool) (*ipnstate.Status, error)
 	MustStatus() *ipnstate.Status
 	Netmap() (*netmap.NetworkMap, error)
@@ -52,6 +56,10 @@ type TailscaleClient interface {
 	ContainerID() string
 	MustID() types.NodeID
 	ReadFile(path string) ([]byte, error)
+	PacketFilter() ([]filter.Match, error)
+	ConnectToNetwork(network *dockertest.Network) error
+	DisconnectFromNetwork(network *dockertest.Network) error
+	ReconnectToNetwork(network *dockertest.Network) error
 
 	// FailingPeersAsString returns a formatted-ish multi-line-string of peers in the client
 	// and a bool indicating if the clients online count and peer count is equal.

integration/tsric/tsric.go

@@ -0,0 +1,334 @@
+// Package tsric provides a TailscaleRustInContainer (tsric) implementation
+// that runs the tailscale-rs axum example inside a Docker container for
+// integration testing with headscale.
+//
+// Unlike tsic (which runs the official Tailscale client), tsric runs a Rust
+// implementation of a Tailscale node. It does not have the `tailscale` CLI,
+// so verification is done externally via headscale API and peer connectivity.
+package tsric
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/juanfont/headscale/hscontrol/util"
+	"github.com/juanfont/headscale/integration/dockertestutil"
+	"github.com/juanfont/headscale/integration/integrationutil"
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+)
+
+const (
+	tsricHashLength = 6
+	caCertRoot      = "/usr/local/share/ca-certificates"
+
+	dockerfileName    = "Dockerfile.tailscale-rs"
+	dockerContextPath = "../."
+
+	buildArgRepo = "TAILSCALE_RS_REPO"
+	buildArgRef  = "TAILSCALE_RS_REF"
+)
+
+// getPrebuiltImage returns the pre-built tailscale-rs Docker image name if set.
+func getPrebuiltImage() string {
+	return os.Getenv("HEADSCALE_INTEGRATION_TAILSCALE_RS_IMAGE")
+}
+
+// TailscaleRustInContainer runs the tailscale-rs axum example as an
+// integration test peer.
+type TailscaleRustInContainer struct {
+	hostname string
+
+	pool      *dockertest.Pool
+	container *dockertest.Resource
+	network   *dockertest.Network
+
+	caCerts      [][]byte
+	headscaleURL string
+	authKey      string
+	extraHosts   []string
+	repo         string
+	ref          string
+}
+
+// Option represents optional settings for a TailscaleRustInContainer instance.
+type Option = func(c *TailscaleRustInContainer)
+
+// WithCACert adds a CA certificate to the trusted certificates of the container.
+func WithCACert(cert []byte) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.caCerts = append(t.caCerts, cert)
+	}
+}
+
+// WithNetwork sets the Docker [dockertest.Network].
+func WithNetwork(network *dockertest.Network) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.network = network
+	}
+}
+
+// WithHeadscaleURL sets the headscale control server URL.
+func WithHeadscaleURL(url string) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.headscaleURL = url
+	}
+}
+
+// WithAuthKey sets the pre-authentication key for joining the tailnet.
+func WithAuthKey(key string) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.authKey = key
+	}
+}
+
+// WithExtraHosts adds extra /etc/hosts entries to the container.
+func WithExtraHosts(hosts []string) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.extraHosts = append(t.extraHosts, hosts...)
+	}
+}
+
+// WithRepo overrides the tailscale-rs git repository URL used by the
+// Dockerfile. Defaults to the public github.com/tailscale/tailscale-rs.
+func WithRepo(url string) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.repo = url
+	}
+}
+
+// WithRef overrides the tailscale-rs git ref (branch, tag, commit) used
+// by the Dockerfile. Defaults to "main".
+func WithRef(ref string) Option {
+	return func(t *TailscaleRustInContainer) {
+		t.ref = ref
+	}
+}
+
+// buildEntrypoint constructs the container entrypoint command.
+//
+// The axum example reads the control URL from TS_CONTROL_URL, the
+// hostname from -H, and the auth key from -k. The key file (-c) is
+// created on first run.
+func (t *TailscaleRustInContainer) buildEntrypoint() []string {
+	var commands []string
+
+	commands = append(commands,
+		"while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
+
+	// CA certs are written by New after the container starts, so the
+	// entrypoint races with that write. Block until the first cert lands.
+	if len(t.caCerts) > 0 {
+		commands = append(commands,
+			fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot))
+	}
+
+	commands = append(commands, "update-ca-certificates 2>/dev/null || true")
+
+	commands = append(commands,
+		fmt.Sprintf(`export TS_CONTROL_URL=%q`, t.headscaleURL),
+		// The tailscale crate refuses to run without this env gate;
+		// see lib.rs in tailscale-rs.
+		"export TS_RS_EXPERIMENT=this_is_unstable_software",
+	)
+
+	axumCmd := "/usr/local/bin/axum -c /tmp/tsrs-keys.json -H " + t.hostname
+	if t.authKey != "" {
+		axumCmd += " -k " + t.authKey
+	}
+
+	commands = append(commands, "exec "+axumCmd)
+
+	return []string{"/bin/sh", "-c", strings.Join(commands, " ; ")}
+}
+
+// New creates and starts a new [TailscaleRustInContainer] instance.
+func New(
+	pool *dockertest.Pool,
+	opts ...Option,
+) (*TailscaleRustInContainer, error) {
+	hash, err := util.GenerateRandomStringDNSSafe(tsricHashLength)
+	if err != nil {
+		return nil, err
+	}
+
+	runID := dockertestutil.GetIntegrationRunID()
+
+	var hostname string
+
+	if runID != "" {
+		runIDShort := runID[len(runID)-6:]
+		hostname = fmt.Sprintf("tsrs-%s-%s", runIDShort, hash)
+	} else {
+		hostname = "tsrs-" + hash
+	}
+
+	t := &TailscaleRustInContainer{
+		hostname: hostname,
+		pool:     pool,
+	}
+
+	for _, opt := range opts {
+		opt(t)
+	}
+
+	if t.network == nil {
+		return nil, errors.New("tsric: no network set") //nolint:err113
+	}
+
+	if t.headscaleURL == "" {
+		return nil, errors.New("tsric: no headscale URL set") //nolint:err113
+	}
+
+	if t.authKey == "" {
+		return nil, errors.New("tsric: no auth key set") //nolint:err113
+	}
+
+	entrypoint := t.buildEntrypoint()
+
+	runOptions := &dockertest.RunOptions{
+		Name:       hostname,
+		Networks:   []*dockertest.Network{t.network},
+		Entrypoint: entrypoint,
+		ExtraHosts: append(t.extraHosts, "host.docker.internal:host-gateway"),
+		Env:        []string{},
+	}
+
+	dockertestutil.DockerAddIntegrationLabels(runOptions, "tailscale-rs")
+
+	err = pool.RemoveContainerByName(hostname)
+	if err != nil {
+		return nil, err
+	}
+
+	var container *dockertest.Resource
+
+	if prebuiltImage := getPrebuiltImage(); prebuiltImage != "" {
+		log.Printf("Using pre-built tailscale-rs image: %s", prebuiltImage)
+
+		repo, tag, ok := strings.Cut(prebuiltImage, ":")
+		if !ok {
+			return nil, fmt.Errorf("tsric: invalid image format %q, expected repository:tag", prebuiltImage) //nolint:err113
+		}
+
+		runOptions.Repository = repo
+		runOptions.Tag = tag
+
+		container, err = pool.RunWithOptions(
+			runOptions,
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerMemoryLimit,
+		)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"tsric: could not start pre-built tailscale-rs container %s: %w",
+				hostname, err,
+			)
+		}
+	} else {
+		// Build from the Dockerfile so callers don't need a local
+		// tailscale-rs checkout; the Dockerfile clones at build time.
+		var buildArgs []docker.BuildArg
+
+		if t.repo != "" {
+			buildArgs = append(buildArgs, docker.BuildArg{Name: buildArgRepo, Value: t.repo})
+		}
+
+		if t.ref != "" {
+			buildArgs = append(buildArgs, docker.BuildArg{Name: buildArgRef, Value: t.ref})
+		}
+
+		buildOptions := &dockertest.BuildOptions{
+			Dockerfile: dockerfileName,
+			ContextDir: dockerContextPath,
+			BuildArgs:  buildArgs,
+		}
+
+		log.Printf("Building tailscale-rs container %s from upstream (this may take a while for the first build)...", hostname)
+
+		container, err = pool.BuildAndRunWithBuildOptions(
+			buildOptions,
+			runOptions,
+			dockertestutil.DockerRestartPolicy,
+			dockertestutil.DockerAllowLocalIPv6,
+			dockertestutil.DockerMemoryLimit,
+		)
+		if err != nil {
+			return nil, fmt.Errorf(
+				"tsric: could not build and start tailscale-rs container %s: %w",
+				hostname, err,
+			)
+		}
+	}
+
+	log.Printf("Created tailscale-rs container %s", hostname)
+
+	t.container = container
+
+	for i, cert := range t.caCerts {
+		err = t.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
+		if err != nil {
+			return nil, fmt.Errorf("writing TLS certificate to container: %w", err)
+		}
+	}
+
+	return t, nil
+}
+
+// Hostname returns the hostname of the [TailscaleRustInContainer] instance.
+func (t *TailscaleRustInContainer) Hostname() string {
+	return t.hostname
+}
+
+// ContainerID returns the Docker container ID.
+func (t *TailscaleRustInContainer) ContainerID() string {
+	return t.container.Container.ID
+}
+
+// Shutdown stops and cleans up the container.
+func (t *TailscaleRustInContainer) Shutdown() (string, string, error) {
+	stdoutPath, stderrPath, err := t.SaveLog("/tmp/control")
+	if err != nil {
+		log.Printf(
+			"saving log from %s: %s",
+			t.hostname,
+			fmt.Errorf("saving log: %w", err),
+		)
+	}
+
+	return stdoutPath, stderrPath, t.pool.Purge(t.container)
+}
+
+// SaveLog saves the current container logs to the given path.
+func (t *TailscaleRustInContainer) SaveLog(path string) (string, string, error) {
+	return dockertestutil.SaveLog(t.pool, t.container, path)
+}
+
+// WriteLogs writes the current stdout/stderr log of the container to
+// the given [io.Writer]s.
+func (t *TailscaleRustInContainer) WriteLogs(stdout, stderr io.Writer) error {
+	return dockertestutil.WriteLog(t.pool, t.container, stdout, stderr)
+}
+
+// Execute runs a command inside the container.
+func (t *TailscaleRustInContainer) Execute(
+	command []string,
+	options ...dockertestutil.ExecuteCommandOption,
+) (string, string, error) {
+	return dockertestutil.ExecuteCommand(
+		t.container,
+		command,
+		[]string{},
+		options...,
+	)
+}
+
+// WriteFile writes a file into the container.
+func (t *TailscaleRustInContainer) WriteFile(path string, data []byte) error {
+	return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
+}

integration/tsric_test.go

@@ -0,0 +1,271 @@
+package integration
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/juanfont/headscale/integration/hsic"
+	"github.com/juanfont/headscale/integration/tsic"
+	"github.com/juanfont/headscale/integration/tsric"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestTailscaleRustAxum tests that the tailscale-rs axum example can join a
+// headscale network and serve HTTP to other peers on the tailnet.
+//
+// Architecture:
+//
+//	headscale (control) <--- tsic (probe client) --curl--> tsric (axum server)
+//
+// The test:
+//  1. Creates a headscale environment with one regular Tailscale client (tsic)
+//  2. Creates a tailscale-rs container running the axum example (tsric)
+//  3. Verifies the tsric node registers with headscale
+//  4. Uses the tsic client to curl the axum web server through the tailnet
+func TestTailscaleRustAxum(t *testing.T) {
+	IntegrationSkip(t)
+
+	// Set up a scenario with one user and one regular Tailscale client.
+	// The regular client acts as a "probe" to verify the tsric node
+	// is reachable on the tailnet.
+	spec := ScenarioSpec{
+		NodesPerUser: 1,
+		Users:        []string{"user1"}, //nolint:goconst // consistent with other integration tests
+	}
+
+	scenario, err := NewScenario(spec)
+
+	require.NoError(t, err)
+	defer scenario.ShutdownAssertNoPanics(t)
+
+	err = scenario.CreateHeadscaleEnv(
+		[]tsic.Option{},
+		hsic.WithTestName("tailscalers"),
+		// The embedded DERP server uses a self-signed cert that
+		// tailscale-rs cannot validate without a custom CA bundle, so
+		// we route DERP through Tailscale's public relays.
+		hsic.WithPublicDERP(),
+		// TODO: drop WithoutTLS once tailscale-rs lets us inject the
+		// headscale CA into its trust chain; until then the control
+		// plane has to be plain HTTP for the Rust client to register.
+		hsic.WithoutTLS(),
+	)
+	requireNoErrHeadscaleEnv(t, err)
+
+	// Get the headscale instance and probe client
+	headscale, err := scenario.Headscale()
+	require.NoError(t, err)
+
+	allClients, err := scenario.ListTailscaleClients()
+	requireNoErrListClients(t, err)
+	require.Len(t, allClients, 1, "expected exactly 1 probe client")
+
+	probeClient := allClients[0]
+
+	// Create auth key for the tailscale-rs node
+	users, err := headscale.ListUsers()
+	require.NoError(t, err)
+	require.NotEmpty(t, users, "expected at least one user")
+
+	var userID uint64
+
+	for _, u := range users {
+		if u.GetName() == "user1" { //nolint:goconst
+			userID = u.GetId()
+
+			break
+		}
+	}
+
+	require.NotZero(t, userID, "user1 not found")
+
+	pak, err := headscale.CreateAuthKey(userID, false, true)
+	require.NoError(t, err)
+
+	// Determine the network and headscale connection details
+	networks := scenario.Networks()
+	require.NotEmpty(t, networks)
+
+	network := networks[0]
+	headscaleIP := headscale.GetIPInNetwork(network)
+	headscaleHostname := headscale.GetHostname()
+	headscaleEndpoint := headscale.GetEndpoint()
+
+	t.Logf("Headscale endpoint: %s (hostname: %s, IP: %s)",
+		headscaleEndpoint, headscaleHostname, headscaleIP)
+
+	// Create the tailscale-rs container
+	tsrsOpts := []tsric.Option{
+		tsric.WithNetwork(network),
+		tsric.WithHeadscaleURL(headscaleEndpoint),
+		tsric.WithAuthKey(pak.GetKey()),
+		tsric.WithExtraHosts([]string{headscaleHostname + ":" + headscaleIP}),
+	}
+
+	cert := headscale.GetCert()
+	if len(cert) > 0 {
+		tsrsOpts = append(tsrsOpts, tsric.WithCACert(cert))
+	}
+
+	t.Log("Creating tailscale-rs container (first build may take several minutes)...")
+
+	tsrs, err := tsric.New(scenario.Pool(), tsrsOpts...)
+	require.NoError(t, err, "failed to create tailscale-rs container")
+
+	defer func() {
+		_, _, err := tsrs.Shutdown()
+		if err != nil {
+			t.Logf("error shutting down tailscale-rs container: %s", err)
+		}
+	}()
+
+	// Wait for the tailscale-rs node to appear in headscale's node list.
+	// Verify it gets both IPv4 and IPv6 addresses and has the expected hostname.
+	var (
+		rustNodeIPv4 string
+		rustNodeIPv6 string
+		rustNodeName string
+	)
+
+	t.Log("Waiting for tailscale-rs node to register with headscale...")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		nodes, err := headscale.ListNodes()
+		assert.NoError(c, err)
+
+		// Expect 2 nodes: 1 tsic probe + 1 tsric
+		assert.GreaterOrEqual(c, len(nodes), 2,
+			"expected at least 2 nodes (1 probe + 1 tailscale-rs)")
+
+		// Find the tailscale-rs node by hostname prefix
+		for _, n := range nodes {
+			if strings.HasPrefix(n.GetGivenName(), "tsrs-") {
+				addrs := n.GetIpAddresses()
+				if len(addrs) > 0 {
+					rustNodeIPv4 = addrs[0]
+				}
+
+				if len(addrs) > 1 {
+					rustNodeIPv6 = addrs[1]
+				}
+
+				rustNodeName = n.GetGivenName()
+			}
+		}
+
+		assert.NotEmpty(c, rustNodeIPv4, "tailscale-rs node should have an IPv4 address")
+	}, 120*time.Second, 2*time.Second, "tailscale-rs node should register with headscale")
+
+	require.NotEmpty(t, rustNodeIPv4, "failed to find tailscale-rs node IP")
+
+	t.Logf("tailscale-rs node %q registered with IPv4=%s IPv6=%s",
+		rustNodeName, rustNodeIPv4, rustNodeIPv6)
+
+	// Verify IPv6 was allocated. The axum example only listens on IPv4,
+	// so we can't curl via IPv6, but headscale should still assign both.
+	assert.NotEmpty(t, rustNodeIPv6,
+		"headscale should assign both IPv4 and IPv6 to the tailscale-rs node")
+
+	// Verify the hostname propagated correctly from the config
+	assert.True(t, strings.HasPrefix(rustNodeName, "tsrs-"),
+		"tailscale-rs node name should start with tsrs- prefix")
+
+	// Verify the probe client sees the tailscale-rs node as a peer
+	t.Log("Verifying probe client sees tailscale-rs as a peer...")
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		status, err := probeClient.Status()
+		assert.NoError(c, err)
+
+		found := false
+
+		for _, peerKey := range status.Peers() {
+			peer := status.Peer[peerKey]
+			if strings.HasPrefix(peer.HostName, "tsrs-") {
+				found = true
+			}
+		}
+
+		assert.True(c, found, "probe client should see tsrs node as a peer")
+	}, 30*time.Second, 2*time.Second, "probe should see tailscale-rs peer in status")
+
+	// Test 1: GET /index.html — verify the axum web server serves content
+	axumURL := fmt.Sprintf("http://%s/index.html", rustNodeIPv4)
+
+	t.Logf("Verifying axum web server is reachable at %s via probe client...", axumURL)
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, err := probeClient.Curl(axumURL)
+		assert.NoError(c, err, "curl to axum server failed")
+		assert.Contains(c, result, "tailscale-rs",
+			"expected index.html to contain 'tailscale-rs'")
+	}, 120*time.Second, 2*time.Second, "axum /index.html should be reachable from probe client")
+
+	t.Log("axum web server is serving content through the tailnet")
+
+	// Test 2: GET /assets/index.css — verify static asset serving works
+	cssURL := fmt.Sprintf("http://%s/assets/index.css", rustNodeIPv4)
+
+	t.Logf("Verifying static asset at %s...", cssURL)
+
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		result, err := probeClient.Curl(cssURL)
+		assert.NoError(c, err, "curl to CSS asset failed")
+		assert.Contains(c, result, "font-family",
+			"expected CSS file to contain 'font-family'")
+	}, 10*time.Second, 1*time.Second, "axum should serve static CSS assets")
+
+	// Test 3: Sequential POST /count — verify the counter increments correctly.
+	// This exercises multiple TCP connections and proves the netstack maintains
+	// state across requests.
+	countURL := fmt.Sprintf("http://%s/count", rustNodeIPv4)
+
+	t.Logf("Verifying /count POST endpoint increments at %s...", countURL)
+
+	// First POST establishes connectivity and gets the initial counter value
+	assert.EventuallyWithT(t, func(c *assert.CollectT) {
+		stdout, _, err := probeClient.Execute([]string{
+			"curl", "--silent",
+			"--connect-timeout", "3",
+			"--max-time", "5",
+			"-X", "POST",
+			countURL,
+		})
+		assert.NoError(c, err, "curl POST to /count failed")
+		assert.Contains(c, stdout, `"count"`,
+			"expected /count response to contain 'count'")
+	}, 30*time.Second, 2*time.Second, "axum /count POST should work")
+
+	// Fire several more POSTs and verify the counter advances.
+	// The axum handler returns {"count": N} where N is the pre-increment value.
+	// After the initial [assert.EventuallyWithT] loop we don't know the exact counter,
+	// but two back-to-back POSTs should return consecutive values.
+	t.Log("Verifying counter increments across multiple requests...")
+
+	var firstCount, secondCount string
+
+	stdout1, _, err := probeClient.Execute([]string{
+		"curl", "--silent", "--max-time", "5", "-X", "POST", countURL,
+	})
+	require.NoError(t, err, "first sequential POST failed")
+
+	firstCount = stdout1
+
+	stdout2, _, err := probeClient.Execute([]string{
+		"curl", "--silent", "--max-time", "5", "-X", "POST", countURL,
+	})
+	require.NoError(t, err, "second sequential POST failed")
+
+	secondCount = stdout2
+
+	t.Logf("Counter responses: first=%s second=%s", firstCount, secondCount)
+
+	// Verify they're different (counter is incrementing)
+	require.NotEqual(t, firstCount, secondCount,
+		"counter should increment between sequential POST requests")
+
+	t.Log("TestTailscaleRustAxum: all checks passed")
+}

integration/utils.go

@@ -1,533 +0,0 @@
-package integration
-
-import (
-	"bufio"
-	"bytes"
-	"fmt"
-	"io"
-	"net/netip"
-	"strings"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/cenkalti/backoff/v5"
-	policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
-	"github.com/juanfont/headscale/hscontrol/util"
-	"github.com/juanfont/headscale/integration/tsic"
-	"github.com/stretchr/testify/assert"
-	"tailscale.com/tailcfg"
-	"tailscale.com/types/ptr"
-)
-
-const (
-	// derpPingTimeout defines the timeout for individual DERP ping operations
-	// Used in DERP connectivity tests to verify relay server communication.
-	derpPingTimeout = 2 * time.Second
-
-	// derpPingCount defines the number of ping attempts for DERP connectivity tests
-	// Higher count provides better reliability assessment of DERP connectivity.
-	derpPingCount = 10
-
-	// TimestampFormat is the standard timestamp format used across all integration tests
-	// Format: "2006-01-02T15-04-05.999999999" provides high precision timestamps
-	// suitable for debugging and log correlation in integration tests.
-	TimestampFormat = "2006-01-02T15-04-05.999999999"
-
-	// TimestampFormatRunID is used for generating unique run identifiers
-	// Format: "20060102-150405" provides compact date-time for file/directory names.
-	TimestampFormatRunID = "20060102-150405"
-)
-
-func assertNoErr(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "unexpected error: %s", err)
-}
-
-func assertNoErrf(t *testing.T, msg string, err error) {
-	t.Helper()
-	if err != nil {
-		t.Fatalf(msg, err)
-	}
-}
-
-func assertNotNil(t *testing.T, thing interface{}) {
-	t.Helper()
-	if thing == nil {
-		t.Fatal("got unexpected nil")
-	}
-}
-
-func assertNoErrHeadscaleEnv(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to create headscale environment: %s", err)
-}
-
-func assertNoErrGetHeadscale(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to get headscale: %s", err)
-}
-
-func assertNoErrListClients(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to list clients: %s", err)
-}
-
-func assertNoErrListClientIPs(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to get client IPs: %s", err)
-}
-
-func assertNoErrSync(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to have all clients sync up: %s", err)
-}
-
-func assertNoErrListFQDN(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to list FQDNs: %s", err)
-}
-
-func assertNoErrLogout(t *testing.T, err error) {
-	t.Helper()
-	assertNoErrf(t, "failed to log out tailscale nodes: %s", err)
-}
-
-func assertContains(t *testing.T, str, subStr string) {
-	t.Helper()
-	if !strings.Contains(str, subStr) {
-		t.Fatalf("%#v does not contain %#v", str, subStr)
-	}
-}
-
-func didClientUseWebsocketForDERP(t *testing.T, client TailscaleClient) bool {
-	t.Helper()
-
-	buf := &bytes.Buffer{}
-	err := client.WriteLogs(buf, buf)
-	if err != nil {
-		t.Fatalf("failed to fetch client logs: %s: %s", client.Hostname(), err)
-	}
-
-	count, err := countMatchingLines(buf, func(line string) bool {
-		return strings.Contains(line, "websocket: connected to ")
-	})
-	if err != nil {
-		t.Fatalf("failed to process client logs: %s: %s", client.Hostname(), err)
-	}
-
-	return count > 0
-}
-
-// pingAllHelper performs ping tests between all clients and addresses, returning success count.
-// This is used to validate network connectivity in integration tests.
-// Returns the total number of successful ping operations.
-func pingAllHelper(t *testing.T, clients []TailscaleClient, addrs []string, opts ...tsic.PingOption) int {
-	t.Helper()
-	success := 0
-
-	for _, client := range clients {
-		for _, addr := range addrs {
-			err := client.Ping(addr, opts...)
-			if err != nil {
-				t.Errorf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
-			} else {
-				success++
-			}
-		}
-	}
-
-	return success
-}
-
-// pingDerpAllHelper performs DERP-based ping tests between all clients and addresses.
-// This specifically tests connectivity through DERP relay servers, which is important
-// for validating NAT traversal and relay functionality. Returns success count.
-func pingDerpAllHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
-	t.Helper()
-	success := 0
-
-	for _, client := range clients {
-		for _, addr := range addrs {
-			if isSelfClient(client, addr) {
-				continue
-			}
-
-			err := client.Ping(
-				addr,
-				tsic.WithPingTimeout(derpPingTimeout),
-				tsic.WithPingCount(derpPingCount),
-				tsic.WithPingUntilDirect(false),
-			)
-			if err != nil {
-				t.Logf("failed to ping %s from %s: %s", addr, client.Hostname(), err)
-			} else {
-				success++
-			}
-		}
-	}
-
-	return success
-}
-
-// assertClientsState validates the status and netmap of a list of
-// clients for the general case of all to all connectivity.
-func assertClientsState(t *testing.T, clients []TailscaleClient) {
-	t.Helper()
-
-	var wg sync.WaitGroup
-
-	for _, client := range clients {
-		wg.Add(1)
-		c := client // Avoid loop pointer
-		go func() {
-			defer wg.Done()
-			assertValidStatus(t, c)
-			assertValidNetcheck(t, c)
-			assertValidNetmap(t, c)
-		}()
-	}
-
-	t.Logf("waiting for client state checks to finish")
-	wg.Wait()
-}
-
-// assertValidNetmap asserts that the netmap of a client has all
-// the minimum required fields set to a known working config for
-// the general case. Fields are checked on self, then all peers.
-// This test is not suitable for ACL/partial connection tests.
-// This test can only be run on clients from 1.56.1. It will
-// automatically pass all clients below that and is safe to call
-// for all versions.
-func assertValidNetmap(t *testing.T, client TailscaleClient) {
-	t.Helper()
-
-	if !util.TailscaleVersionNewerOrEqual("1.56", client.Version()) {
-		t.Logf("%q has version %q, skipping netmap check...", client.Hostname(), client.Version())
-
-		return
-	}
-
-	t.Logf("Checking netmap of %q", client.Hostname())
-
-	netmap, err := client.Netmap()
-	if err != nil {
-		t.Fatalf("getting netmap for %q: %s", client.Hostname(), err)
-	}
-
-	assert.Truef(t, netmap.SelfNode.Hostinfo().Valid(), "%q does not have Hostinfo", client.Hostname())
-	if hi := netmap.SelfNode.Hostinfo(); hi.Valid() {
-		assert.LessOrEqual(t, 1, netmap.SelfNode.Hostinfo().Services().Len(), "%q does not have enough services, got: %v", client.Hostname(), netmap.SelfNode.Hostinfo().Services())
-	}
-
-	assert.NotEmptyf(t, netmap.SelfNode.AllowedIPs(), "%q does not have any allowed IPs", client.Hostname())
-	assert.NotEmptyf(t, netmap.SelfNode.Addresses(), "%q does not have any addresses", client.Hostname())
-
-	assert.Truef(t, netmap.SelfNode.Online().Get(), "%q is not online", client.Hostname())
-
-	assert.Falsef(t, netmap.SelfNode.Key().IsZero(), "%q does not have a valid NodeKey", client.Hostname())
-	assert.Falsef(t, netmap.SelfNode.Machine().IsZero(), "%q does not have a valid MachineKey", client.Hostname())
-	assert.Falsef(t, netmap.SelfNode.DiscoKey().IsZero(), "%q does not have a valid DiscoKey", client.Hostname())
-
-	for _, peer := range netmap.Peers {
-		assert.NotEqualf(t, "127.3.3.40:0", peer.LegacyDERPString(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.LegacyDERPString())
-		assert.NotEqualf(t, 0, peer.HomeDERP(), "peer (%s) has no home DERP in %q's netmap, got: %d", peer.ComputedName(), client.Hostname(), peer.HomeDERP())
-
-		assert.Truef(t, peer.Hostinfo().Valid(), "peer (%s) of %q does not have Hostinfo", peer.ComputedName(), client.Hostname())
-		if hi := peer.Hostinfo(); hi.Valid() {
-			assert.LessOrEqualf(t, 3, peer.Hostinfo().Services().Len(), "peer (%s) of %q does not have enough services, got: %v", peer.ComputedName(), client.Hostname(), peer.Hostinfo().Services())
-
-			// Netinfo is not always set
-			// assert.Truef(t, hi.NetInfo().Valid(), "peer (%s) of %q does not have NetInfo", peer.ComputedName(), client.Hostname())
-			if ni := hi.NetInfo(); ni.Valid() {
-				assert.NotEqualf(t, 0, ni.PreferredDERP(), "peer (%s) has no home DERP in %q's netmap, got: %s", peer.ComputedName(), client.Hostname(), peer.Hostinfo().NetInfo().PreferredDERP())
-			}
-		}
-
-		assert.NotEmptyf(t, peer.Endpoints(), "peer (%s) of %q does not have any endpoints", peer.ComputedName(), client.Hostname())
-		assert.NotEmptyf(t, peer.AllowedIPs(), "peer (%s) of %q does not have any allowed IPs", peer.ComputedName(), client.Hostname())
-		assert.NotEmptyf(t, peer.Addresses(), "peer (%s) of %q does not have any addresses", peer.ComputedName(), client.Hostname())
-
-		assert.Truef(t, peer.Online().Get(), "peer (%s) of %q is not online", peer.ComputedName(), client.Hostname())
-
-		assert.Falsef(t, peer.Key().IsZero(), "peer (%s) of %q does not have a valid NodeKey", peer.ComputedName(), client.Hostname())
-		assert.Falsef(t, peer.Machine().IsZero(), "peer (%s) of %q does not have a valid MachineKey", peer.ComputedName(), client.Hostname())
-		assert.Falsef(t, peer.DiscoKey().IsZero(), "peer (%s) of %q does not have a valid DiscoKey", peer.ComputedName(), client.Hostname())
-	}
-}
-
-// assertValidStatus asserts that the status of a client has all
-// the minimum required fields set to a known working config for
-// the general case. Fields are checked on self, then all peers.
-// This test is not suitable for ACL/partial connection tests.
-func assertValidStatus(t *testing.T, client TailscaleClient) {
-	t.Helper()
-	status, err := client.Status(true)
-	if err != nil {
-		t.Fatalf("getting status for %q: %s", client.Hostname(), err)
-	}
-
-	assert.NotEmptyf(t, status.Self.HostName, "%q does not have HostName set, likely missing Hostinfo", client.Hostname())
-	assert.NotEmptyf(t, status.Self.OS, "%q does not have OS set, likely missing Hostinfo", client.Hostname())
-	assert.NotEmptyf(t, status.Self.Relay, "%q does not have a relay, likely missing Hostinfo/Netinfo", client.Hostname())
-
-	assert.NotEmptyf(t, status.Self.TailscaleIPs, "%q does not have Tailscale IPs", client.Hostname())
-
-	// This seem to not appear until version 1.56
-	if status.Self.AllowedIPs != nil {
-		assert.NotEmptyf(t, status.Self.AllowedIPs, "%q does not have any allowed IPs", client.Hostname())
-	}
-
-	assert.NotEmptyf(t, status.Self.Addrs, "%q does not have any endpoints", client.Hostname())
-
-	assert.Truef(t, status.Self.Online, "%q is not online", client.Hostname())
-
-	assert.Truef(t, status.Self.InNetworkMap, "%q is not in network map", client.Hostname())
-
-	// This isn't really relevant for Self as it won't be in its own socket/wireguard.
-	// assert.Truef(t, status.Self.InMagicSock, "%q is not tracked by magicsock", client.Hostname())
-	// assert.Truef(t, status.Self.InEngine, "%q is not in wireguard engine", client.Hostname())
-
-	for _, peer := range status.Peer {
-		assert.NotEmptyf(t, peer.HostName, "peer (%s) of %q does not have HostName set, likely missing Hostinfo", peer.DNSName, client.Hostname())
-		assert.NotEmptyf(t, peer.OS, "peer (%s) of %q does not have OS set, likely missing Hostinfo", peer.DNSName, client.Hostname())
-		assert.NotEmptyf(t, peer.Relay, "peer (%s) of %q does not have a relay, likely missing Hostinfo/Netinfo", peer.DNSName, client.Hostname())
-
-		assert.NotEmptyf(t, peer.TailscaleIPs, "peer (%s) of %q does not have Tailscale IPs", peer.DNSName, client.Hostname())
-
-		// This seem to not appear until version 1.56
-		if peer.AllowedIPs != nil {
-			assert.NotEmptyf(t, peer.AllowedIPs, "peer (%s) of %q does not have any allowed IPs", peer.DNSName, client.Hostname())
-		}
-
-		// Addrs does not seem to appear in the status from peers.
-		// assert.NotEmptyf(t, peer.Addrs, "peer (%s) of %q does not have any endpoints", peer.DNSName, client.Hostname())
-
-		assert.Truef(t, peer.Online, "peer (%s) of %q is not online", peer.DNSName, client.Hostname())
-
-		assert.Truef(t, peer.InNetworkMap, "peer (%s) of %q is not in network map", peer.DNSName, client.Hostname())
-		assert.Truef(t, peer.InMagicSock, "peer (%s) of %q is not tracked by magicsock", peer.DNSName, client.Hostname())
-
-		// TODO(kradalby): InEngine is only true when a proper tunnel is set up,
-		// there might be some interesting stuff to test here in the future.
-		// assert.Truef(t, peer.InEngine, "peer (%s) of %q is not in wireguard engine", peer.DNSName, client.Hostname())
-	}
-}
-
-func assertValidNetcheck(t *testing.T, client TailscaleClient) {
-	t.Helper()
-	report, err := client.Netcheck()
-	if err != nil {
-		t.Fatalf("getting status for %q: %s", client.Hostname(), err)
-	}
-
-	assert.NotEqualf(t, 0, report.PreferredDERP, "%q does not have a DERP relay", client.Hostname())
-}
-
-// assertCommandOutputContains executes a command with exponential backoff retry until the output
-// contains the expected string or timeout is reached (10 seconds).
-// This implements eventual consistency patterns and should be used instead of time.Sleep
-// before executing commands that depend on network state propagation.
-//
-// Timeout: 10 seconds with exponential backoff
-// Use cases: DNS resolution, route propagation, policy updates.
-func assertCommandOutputContains(t *testing.T, c TailscaleClient, command []string, contains string) {
-	t.Helper()
-
-	_, err := backoff.Retry(t.Context(), func() (struct{}, error) {
-		stdout, stderr, err := c.Execute(command)
-		if err != nil {
-			return struct{}{}, fmt.Errorf("executing command, stdout: %q stderr: %q, err: %w", stdout, stderr, err)
-		}
-
-		if !strings.Contains(stdout, contains) {
-			return struct{}{}, fmt.Errorf("executing command, expected string %q not found in %q", contains, stdout)
-		}
-
-		return struct{}{}, nil
-	}, backoff.WithBackOff(backoff.NewExponentialBackOff()), backoff.WithMaxElapsedTime(10*time.Second))
-
-	assert.NoError(t, err)
-}
-
-func isSelfClient(client TailscaleClient, addr string) bool {
-	if addr == client.Hostname() {
-		return true
-	}
-
-	ips, err := client.IPs()
-	if err != nil {
-		return false
-	}
-
-	for _, ip := range ips {
-		if ip.String() == addr {
-			return true
-		}
-	}
-
-	return false
-}
-
-func dockertestMaxWait() time.Duration {
-	wait := 300 * time.Second //nolint
-
-	if util.IsCI() {
-		wait = 600 * time.Second //nolint
-	}
-
-	return wait
-}
-
-func countMatchingLines(in io.Reader, predicate func(string) bool) (int, error) {
-	count := 0
-	scanner := bufio.NewScanner(in)
-	{
-		const logBufferInitialSize = 1024 << 10 // preallocate 1 MiB
-		buff := make([]byte, logBufferInitialSize)
-		scanner.Buffer(buff, len(buff))
-		scanner.Split(bufio.ScanLines)
-	}
-
-	for scanner.Scan() {
-		if predicate(scanner.Text()) {
-			count += 1
-		}
-	}
-
-	return count, scanner.Err()
-}
-
-// func dockertestCommandTimeout() time.Duration {
-// 	timeout := 10 * time.Second //nolint
-//
-// 	if isCI() {
-// 		timeout = 60 * time.Second //nolint
-// 	}
-//
-// 	return timeout
-// }
-
-// pingAllNegativeHelper is intended to have 1 or more nodes timing out from the ping,
-// it counts failures instead of successes.
-// func pingAllNegativeHelper(t *testing.T, clients []TailscaleClient, addrs []string) int {
-// 	t.Helper()
-// 	failures := 0
-//
-// 	timeout := 100
-// 	count := 3
-//
-// 	for _, client := range clients {
-// 		for _, addr := range addrs {
-// 			err := client.Ping(
-// 				addr,
-// 				tsic.WithPingTimeout(time.Duration(timeout)*time.Millisecond),
-// 				tsic.WithPingCount(count),
-// 			)
-// 			if err != nil {
-// 				failures++
-// 			}
-// 		}
-// 	}
-//
-// 	return failures
-// }
-
-// // findPeerByIP takes an IP and a map of peers from status.Peer, and returns a *ipnstate.PeerStatus
-// // if there is a peer with the given IP. If no peer is found, nil is returned.
-// func findPeerByIP(
-// 	ip netip.Addr,
-// 	peers map[key.NodePublic]*ipnstate.PeerStatus,
-// ) *ipnstate.PeerStatus {
-// 	for _, peer := range peers {
-// 		for _, peerIP := range peer.TailscaleIPs {
-// 			if ip == peerIP {
-// 				return peer
-// 			}
-// 		}
-// }
-//
-// 	return nil
-// }
-
-// Helper functions for creating typed policy entities
-
-// wildcard returns a wildcard alias (*).
-func wildcard() policyv2.Alias {
-	return policyv2.Wildcard
-}
-
-// usernamep returns a pointer to a Username as an Alias.
-func usernamep(name string) policyv2.Alias {
-	return ptr.To(policyv2.Username(name))
-}
-
-// hostp returns a pointer to a Host.
-func hostp(name string) policyv2.Alias {
-	return ptr.To(policyv2.Host(name))
-}
-
-// groupp returns a pointer to a Group as an Alias.
-func groupp(name string) policyv2.Alias {
-	return ptr.To(policyv2.Group(name))
-}
-
-// tagp returns a pointer to a Tag as an Alias.
-func tagp(name string) policyv2.Alias {
-	return ptr.To(policyv2.Tag(name))
-}
-
-// prefixp returns a pointer to a Prefix from a CIDR string.
-func prefixp(cidr string) policyv2.Alias {
-	prefix := netip.MustParsePrefix(cidr)
-	return ptr.To(policyv2.Prefix(prefix))
-}
-
-// aliasWithPorts creates an AliasWithPorts structure from an alias and ports.
-func aliasWithPorts(alias policyv2.Alias, ports ...tailcfg.PortRange) policyv2.AliasWithPorts {
-	return policyv2.AliasWithPorts{
-		Alias: alias,
-		Ports: ports,
-	}
-}
-
-// usernameOwner returns a Username as an Owner for use in TagOwners.
-func usernameOwner(name string) policyv2.Owner {
-	return ptr.To(policyv2.Username(name))
-}
-
-// groupOwner returns a Group as an Owner for use in TagOwners.
-func groupOwner(name string) policyv2.Owner {
-	return ptr.To(policyv2.Group(name))
-}
-
-// usernameApprover returns a Username as an AutoApprover.
-func usernameApprover(name string) policyv2.AutoApprover {
-	return ptr.To(policyv2.Username(name))
-}
-
-// groupApprover returns a Group as an AutoApprover.
-func groupApprover(name string) policyv2.AutoApprover {
-	return ptr.To(policyv2.Group(name))
-}
-
-// tagApprover returns a Tag as an AutoApprover.
-func tagApprover(name string) policyv2.AutoApprover {
-	return ptr.To(policyv2.Tag(name))
-}
-
-//
-// // findPeerByHostname takes a hostname and a map of peers from status.Peer, and returns a *ipnstate.PeerStatus
-// // if there is a peer with the given hostname. If no peer is found, nil is returned.
-// func findPeerByHostname(
-// 	hostname string,
-// 	peers map[key.NodePublic]*ipnstate.PeerStatus,
-// ) *ipnstate.PeerStatus {
-// 	for _, peer := range peers {
-// 		if hostname == peer.HostName {
-// 			return peer
-// 		}
-// 	}
-//
-// 	return nil
-// }