From 66a5f99bfaa95fb42e28ac390a258207972621ea Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 22 May 2026 09:45:38 +0000 Subject: [PATCH 01/10] gh: pre-pull released tailscale images for fork-PR CI Fork PRs anonymous-pull tailscale/tailscale:vX.Y at test time and hit Docker Hub rate limits, causing flakes. A new build-tailscale-released job pulls the MustTestVersions set from ghcr.io once per CI run and ships them to every test job as a gzipped tarball artifact, matching the shape of the existing headscale/HEAD/postgres caches. The version list is driven by 'hi list-versions' so capver is the single source of truth; adding a version there propagates to CI without YAML edits. ghcr.io public reads need no auth, so the new job has no docker login step. --- .../workflows/integration-test-template.yml | 6 ++ .github/workflows/test-integration.yaml | 55 ++++++++++++++++++- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test-template.yml b/.github/workflows/integration-test-template.yml index 427f63e2..68dbfd36 100644 --- a/.github/workflows/integration-test-template.yml +++ b/.github/workflows/integration-test-template.yml @@ -51,6 +51,11 @@ jobs: with: name: tailscale-head-image path: /tmp/artifacts + - name: Download tailscale released images + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: tailscale-released-images + path: /tmp/artifacts - name: Download hi binary uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: @@ -86,6 +91,7 @@ jobs: run: | gunzip -c /tmp/artifacts/headscale-image.tar.gz | docker load gunzip -c /tmp/artifacts/tailscale-head-image.tar.gz | docker load + gunzip -c /tmp/artifacts/tailscale-released-images.tar.gz | docker load if [ -f /tmp/artifacts/postgres-image.tar.gz ]; then gunzip -c /tmp/artifacts/postgres-image.tar.gz | docker load fi diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index f257ae0e..e99cafcc 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -9,6 +9,9 @@ concurrency: jobs: # build: Builds binaries and Docker images once, uploads as artifacts for reuse. # build-postgres: Pulls postgres image separately to avoid Docker Hub rate limits. + # build-tailscale-released: Pre-pulls released Tailscale images from ghcr.io + # so fork PRs (no DOCKERHUB_USERNAME secret) don't hit Docker Hub rate + # limits at test time. # sqlite: Runs all integration tests with SQLite backend. # postgres: Runs a subset of tests with PostgreSQL to verify database compatibility. build: @@ -150,9 +153,57 @@ jobs: name: postgres-image path: postgres-image.tar.gz retention-days: 10 - sqlite: + build-tailscale-released: + runs-on: ubuntu-24.04-arm needs: build if: needs.build.outputs.files-changed == 'true' + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + - uses: nix-community/cache-nix-action@135667ec418502fa5a3598af6fb9eb733888ce6a # v6.1.3 + with: + primary-key: nix-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} + restore-prefixes-first-match: nix-${{ runner.os }}-${{ runner.arch }} + - name: Force overlay2 storage driver + run: | + sudo mkdir -p /etc/docker + echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json + sudo systemctl restart docker + docker version + - name: List Tailscale versions to pre-pull + id: versions + run: | + versions=$(nix develop --command go run ./cmd/hi list-versions --set=must --exclude=head) + echo "versions=${versions}" >> "$GITHUB_OUTPUT" + echo "Pre-pulling: ${versions}" + - name: Pull released Tailscale images + run: | + # ghcr.io public reads are anonymous and unmetered, so no docker + # login is needed even on fork PRs without DOCKERHUB_USERNAME. + # Pull in parallel; xargs -P 0 fans out one process per tag and + # returns non-zero if any pull fails. + echo "${{ steps.versions.outputs.versions }}" \ + | tr ' ' '\n' \ + | xargs -P 0 -I{} docker pull "ghcr.io/tailscale/tailscale:{}" + - name: Save Tailscale images to tarball + run: | + # Single docker save with all refs: one consistent snapshot, no + # parallel-daemon race. + refs="" + for v in ${{ steps.versions.outputs.versions }}; do + refs="${refs} ghcr.io/tailscale/tailscale:${v}" + done + docker save ${refs} | gzip > tailscale-released-images.tar.gz + ls -lh tailscale-released-images.tar.gz + - name: Upload Tailscale released images + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 + with: + name: tailscale-released-images + path: tailscale-released-images.tar.gz + retention-days: 10 + sqlite: + needs: [build, build-tailscale-released] + if: needs.build.outputs.files-changed == 'true' strategy: fail-fast: false matrix: @@ -309,7 +360,7 @@ jobs: postgres_flag: "--postgres=0" database_name: "sqlite" postgres: - needs: [build, build-postgres] + needs: [build, build-postgres, build-tailscale-released] if: needs.build.outputs.files-changed == 'true' strategy: fail-fast: false From 4483fd0cad38717913e7509fc50f9d48c691b02b Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 22 May 2026 10:35:20 +0000 Subject: [PATCH 02/10] tsic, gh: keep unstable on Docker Hub ghcr.io/tailscale/tailscale:unstable is stale (last updated 2022, points to v1.35.25). Only tailscale/tailscale on Docker Hub publishes current unstable builds, so tsic.go reverts the unstable case and build-tailscale-released pulls unstable from Docker Hub while keeping release tags on ghcr.io. Release tags on ghcr.io match Docker Hub by digest (verified v1.96), so the rate-limit avoidance for the bulk of pulls is preserved. Add the conditional docker/login-action step to the new job so the main repo gets authenticated pulls; fork PRs fall through to anonymous DH for the single unstable pull per CI run, well under the 100/6h anonymous limit. --- .github/workflows/test-integration.yaml | 40 +++++++++++++++++-------- integration/tsic/tsic.go | 5 +++- 2 files changed, 31 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-integration.yaml b/.github/workflows/test-integration.yaml index e99cafcc..f1532a00 100644 --- a/.github/workflows/test-integration.yaml +++ b/.github/workflows/test-integration.yaml @@ -170,30 +170,44 @@ jobs: echo '{"storage-driver":"overlay2"}' | sudo tee /etc/docker/daemon.json sudo systemctl restart docker docker version + - name: Login to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_CI_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_CI_TOKEN }} + if: env.DOCKERHUB_USERNAME != '' + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ env.DOCKERHUB_USERNAME }} + password: ${{ env.DOCKERHUB_TOKEN }} - name: List Tailscale versions to pre-pull id: versions run: | versions=$(nix develop --command go run ./cmd/hi list-versions --set=must --exclude=head) echo "versions=${versions}" >> "$GITHUB_OUTPUT" echo "Pre-pulling: ${versions}" - - name: Pull released Tailscale images + - name: Pull Tailscale images run: | - # ghcr.io public reads are anonymous and unmetered, so no docker - # login is needed even on fork PRs without DOCKERHUB_USERNAME. - # Pull in parallel; xargs -P 0 fans out one process per tag and - # returns non-zero if any pull fails. - echo "${{ steps.versions.outputs.versions }}" \ - | tr ' ' '\n' \ - | xargs -P 0 -I{} docker pull "ghcr.io/tailscale/tailscale:{}" + # Releases come from ghcr.io (anonymous, unmetered). The + # "unstable" floating tag on ghcr.io has been stale since 2022, + # so it still needs to come from Docker Hub. xargs -P 0 fans + # out one process per tag and returns non-zero if any pull + # fails. + refs="" + for v in ${{ steps.versions.outputs.versions }}; do + if [ "${v}" = "unstable" ]; then + refs="${refs} tailscale/tailscale:${v}" + else + refs="${refs} ghcr.io/tailscale/tailscale:${v}" + fi + done + echo "${refs}" | tr ' ' '\n' | grep -v '^$' \ + | xargs -P 0 -I{} docker pull "{}" + echo "REFS=${refs}" >> "$GITHUB_ENV" - name: Save Tailscale images to tarball run: | # Single docker save with all refs: one consistent snapshot, no # parallel-daemon race. - refs="" - for v in ${{ steps.versions.outputs.versions }}; do - refs="${refs} ghcr.io/tailscale/tailscale:${v}" - done - docker save ${refs} | gzip > tailscale-released-images.tar.gz + docker save ${REFS} | gzip > tailscale-released-images.tar.gz ls -lh tailscale-released-images.tar.gz - name: Upload Tailscale released images uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index 1c99f41a..9893f5d5 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -522,7 +522,10 @@ func New( } } case "unstable": - tailscaleOptions.Repository = "ghcr.io/tailscale/tailscale" + // ghcr.io/tailscale/tailscale:unstable is stale (last updated + // 2022); only tailscale/tailscale on Docker Hub publishes + // current unstable builds. + tailscaleOptions.Repository = "tailscale/tailscale" tailscaleOptions.Tag = version err = dockertestutil.PullWithAuth(pool, tailscaleOptions.Repository+":"+tailscaleOptions.Tag) From 77ba225cdbfbdca984adaaca56565158624c95d9 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Tue, 26 May 2026 13:55:07 +0000 Subject: [PATCH 03/10] db: treat Go module pseudo-versions as dev builds Untagged main-sha builds inherit a Go module pseudo-version from runtime/debug.BuildInfo (vX.Y.Z--). isDev only filtered "", "dev", and "(devel)", so the pseudo-version was stored in database_versions and the next real release tripped the multi-minor upgrade guard: headscale version v0.29.0-beta.1 cannot be used with a database last used by v0.0.0-20260520093041-e4e742c776ee, upgrading more than one minor version at a time is not supported Add pseudoVersionTime, a regex + time.Parse predicate covering all three Go pseudo-version forms (v0.0.0 base, pre-release ancestor, release ancestor), and delegate from isDev. The dev gate at db.go:790 already prevents pseudo-versions from being written, so already-poisoned databases self-heal on the next real-release start. Fixes #3281 --- hscontrol/db/versioncheck.go | 44 ++++++- hscontrol/db/versioncheck_test.go | 205 +++++++++++++++++++++++++++++- 2 files changed, 242 insertions(+), 7 deletions(-) diff --git a/hscontrol/db/versioncheck.go b/hscontrol/db/versioncheck.go index 5c028f43..a071348c 100644 --- a/hscontrol/db/versioncheck.go +++ b/hscontrol/db/versioncheck.go @@ -3,6 +3,7 @@ package db import ( "errors" "fmt" + "regexp" "strconv" "strings" "time" @@ -139,10 +140,49 @@ func setDatabaseVersion(db *gorm.DB, version string) error { return nil } +// pseudoVersionTimeLayout is Go's pseudo-version timestamp layout +// (golang.org/ref/mod#pseudo-versions): UTC yyyymmddhhmmss. +const pseudoVersionTimeLayout = "20060102150405" + +// pseudoVersionSuffix matches the trailing "<14 digits>-<12 +// lowercase hex>" of a Go module pseudo-version. The base form +// (vX.0.0--) uses "-" before the timestamp; the +// pre-release-ancestor and release-ancestor forms +// (vX.Y.Z-pre.0.- and vX.Y.(Z+1)-0.-) use "." +// because the digit-only "0" marker precedes the timestamp. +var pseudoVersionSuffix = regexp.MustCompile(`[-.](\d{14})-[0-9a-f]{12}$`) + +// pseudoVersionTime returns the embedded commit time when v is a +// syntactically and semantically valid Go module pseudo-version. The +// timestamp must parse as a real UTC time; lookalikes with malformed +// dates (e.g. month 13, day 30 in February) are rejected. +func pseudoVersionTime(v string) (time.Time, bool) { + m := pseudoVersionSuffix.FindStringSubmatch(v) + if m == nil { + return time.Time{}, false + } + + t, err := time.Parse(pseudoVersionTimeLayout, m[1]) + if err != nil { + return time.Time{}, false + } + + return t, true +} + // isDev reports whether a version string represents a development build -// that should skip version checking. +// that should skip version checking. Go module pseudo-versions (used by +// untagged main-sha builds, where runtime/debug.BuildInfo falls back to +// vX.Y.Z--) are treated as dev to avoid poisoning +// database_versions with synthetic baselines. func isDev(version string) bool { - return version == "" || version == "dev" || version == "(devel)" + if version == "" || version == "dev" || version == "(devel)" { + return true + } + + _, ok := pseudoVersionTime(version) + + return ok } // checkVersionUpgradePath verifies that the running headscale version diff --git a/hscontrol/db/versioncheck_test.go b/hscontrol/db/versioncheck_test.go index 83c9ab5c..87089ca5 100644 --- a/hscontrol/db/versioncheck_test.go +++ b/hscontrol/db/versioncheck_test.go @@ -3,6 +3,7 @@ package db import ( "fmt" "testing" + "time" "github.com/glebarez/sqlite" "github.com/stretchr/testify/assert" @@ -56,12 +57,206 @@ func TestSemverString(t *testing.T) { assert.Equal(t, "v0.28.3", s.String()) } +func TestPseudoVersionTime(t *testing.T) { + parseTS := func(s string) time.Time { + t.Helper() + + ts, err := time.Parse(pseudoVersionTimeLayout, s) + require.NoError(t, err) + + return ts + } + + tests := []struct { + name string + input string + wantOK bool + wantTime time.Time + }{ + // Accept: all three Go pseudo-version shapes. + { + name: "no ancestor tag (v0.0.0 base)", + input: "v0.0.0-20260522092201-58a85b68b3d9", + wantOK: true, + wantTime: parseTS("20260522092201"), + }, + { + name: "ancestor is pre-release tag", + input: "v0.29.0-beta.1.0.20260522092201-58a85b68b3d9", + wantOK: true, + wantTime: parseTS("20260522092201"), + }, + { + name: "ancestor is release tag", + input: "v0.29.1-0.20260522092201-58a85b68b3d9", + wantOK: true, + wantTime: parseTS("20260522092201"), + }, + { + name: "earliest realistic Go module date", + input: "v0.0.0-20180101000000-000000000000", + wantOK: true, + wantTime: parseTS("20180101000000"), + }, + + // Reject: real release tags must not look like pseudo-versions. + {name: "release tag", input: "v0.29.0"}, + {name: "pre-release tag", input: "v0.29.0-beta.1"}, + {name: "rc tag", input: "v0.29.0-rc1"}, + {name: "tag with build metadata", input: "v0.29.0+build123"}, + + // Reject: literals handled elsewhere. + {name: "empty", input: ""}, + {name: "dev literal", input: "dev"}, + {name: "devel literal", input: "(devel)"}, + + // Reject: malformed hash. + {name: "hash too short", input: "v0.0.0-20260522092201-58a85b6"}, + {name: "hash too long", input: "v0.0.0-20260522092201-58a85b68b3d9aa"}, + {name: "hash uppercase hex", input: "v0.0.0-20260522092201-58A85B68B3D9"}, + {name: "hash non-hex", input: "v0.0.0-20260522092201-zzzzzzzzzzzz"}, + + // Reject: malformed timestamp. + {name: "timestamp too short", input: "v0.0.0-2026052209220-58a85b68b3d9"}, + {name: "timestamp too long", input: "v0.0.0-202605220922010-58a85b68b3d9"}, + {name: "invalid month", input: "v0.0.0-20261322092201-58a85b68b3d9"}, + {name: "invalid day", input: "v0.0.0-20260230092201-58a85b68b3d9"}, + {name: "invalid hour", input: "v0.0.0-20260522252201-58a85b68b3d9"}, + {name: "invalid minute", input: "v0.0.0-20260522096001-58a85b68b3d9"}, + {name: "invalid second", input: "v0.0.0-20260522092260-58a85b68b3d9"}, + {name: "leap day on non-leap year", input: "v0.0.0-20230229000000-58a85b68b3d9"}, + + // Reject: missing components. + {name: "missing date and hash", input: "v0.0.0-"}, + {name: "missing hash", input: "v0.0.0-20260522092201-"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := pseudoVersionTime(tt.input) + assert.Equal(t, tt.wantOK, ok) + + if tt.wantOK { + assert.True(t, got.Equal(tt.wantTime), + "want %s, got %s", tt.wantTime, got) + } + }) + } +} + func TestIsDev(t *testing.T) { - assert.True(t, isDev("")) - assert.True(t, isDev("dev")) - assert.True(t, isDev("(devel)")) - assert.False(t, isDev("v0.28.0")) - assert.False(t, isDev("0.28.0")) + tests := []struct { + name string + input string + want bool + }{ + // Existing literals. + {name: "empty", input: "", want: true}, + {name: "dev", input: "dev", want: true}, + {name: "devel", input: "(devel)", want: true}, + {name: "release tag", input: "v0.28.0", want: false}, + {name: "release tag no v", input: "0.28.0", want: false}, + {name: "pre-release tag", input: "v0.29.0-beta.1", want: false}, + + // Go module pseudo-versions — all three shapes Go emits per + // golang.org/ref/mod#pseudo-versions. Untagged commits + // (such as main-sha docker builds) must be treated as dev + // so they neither poison database_versions nor trip the + // upgrade-path guard. + { + name: "pseudo v0.0.0 base", + input: "v0.0.0-20260522092201-58a85b68b3d9", + want: true, + }, + { + name: "pseudo from pre-release ancestor", + input: "v0.29.0-beta.1.0.20260522092201-58a85b68b3d9", + want: true, + }, + { + name: "pseudo from release ancestor", + input: "v0.29.1-0.20260522092201-58a85b68b3d9", + want: true, + }, + + // Malformed pseudo-version lookalikes must NOT be treated + // as dev — they fall through to the semver parser. + { + name: "malformed timestamp not dev", + input: "v0.0.0-20261322092201-58a85b68b3d9", + want: false, + }, + { + name: "hash wrong length not dev", + input: "v0.0.0-20260522092201-58a85b6", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isDev(tt.input)) + }) + } +} + +// TestCheckVersionUpgradePath_StoredPseudoVersion exercises the +// upgrade path when database_versions holds a Go module pseudo-version +// written by an untagged main-sha build. Without dev handling, the +// stored pseudo-version parses as v0.0.0 and the next real release +// trips the multi-minor guard. +func TestCheckVersionUpgradePath_StoredPseudoVersion(t *testing.T) { + tests := []struct { + name string + stored string + currentVersion string + }{ + { + name: "v0.0.0 base pseudo to real release", + stored: "v0.0.0-20260520093041-e4e742c776ee", + currentVersion: "v0.29.0-beta.1", + }, + { + name: "pseudo from pre-release ancestor", + stored: "v0.29.0-beta.1.0.20260520093041-e4e742c776ee", + currentVersion: "v0.29.0", + }, + { + name: "pseudo from release ancestor", + stored: "v0.28.1-0.20260520093041-e4e742c776ee", + currentVersion: "v0.29.0", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db := versionTestDB(t) + require.NoError(t, setDatabaseVersion(db, tt.stored)) + err := checkVersionUpgradePathFromVersions(db, tt.currentVersion) + assert.NoError(t, err) + }) + } +} + +// TestCheckVersionUpgradePath_CurrentPseudoDoesNotPoison locks the +// contract that a main-sha (pseudo-version) binary must preserve the +// stored real release so the next real release can upgrade cleanly. +// Mirrors the gating in db.go around setDatabaseVersion. +func TestCheckVersionUpgradePath_CurrentPseudoDoesNotPoison(t *testing.T) { + db := versionTestDB(t) + require.NoError(t, setDatabaseVersion(db, "v0.28.0")) + + current := "v0.0.0-20260522092201-58a85b68b3d9" + err := checkVersionUpgradePathFromVersions(db, current) + require.NoError(t, err) + + // Mirror db.go: only write the current version when !isDev. + if !isDev(current) { + require.NoError(t, setDatabaseVersion(db, current)) + } + + stored, err := getDatabaseVersion(db) + require.NoError(t, err) + assert.Equal(t, "v0.28.0", stored, + "pseudo-version run must not overwrite stored release") } // versionTestDB creates an in-memory SQLite database with the From ea8fc725700918a8aba59f10d6ba2469a73e82ad Mon Sep 17 00:00:00 2001 From: Shourya Gautam Date: Fri, 22 May 2026 21:56:30 +0530 Subject: [PATCH 04/10] db: backfill zero-time node expiry to NULL Versions before 0.28 persisted a pointer to a zero time.Time as '0001-01-01 00:00:00+00:00' in nodes.expiry rather than NULL. 0.29 reports those rows as expired. #3197 and #3199 stopped new writes; this normalises the existing rows so the column once again means "no expiry" when unset. Fixes: #3284 Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 2 +- hscontrol/db/db.go | 22 +++++ hscontrol/db/db_test.go | 54 ++++++++++++ .../zero_time_expiry_migration_test.sql | 82 +++++++++++++++++++ 4 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 hscontrol/db/testdata/sqlite/zero_time_expiry_migration_test.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index 90141966..89f7337c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -306,8 +306,8 @@ connected" routers that maintain their control session but cannot route packets. - Remove old migrations for the debian package [#3185](https://github.com/juanfont/headscale/pull/3185) - Install `config-example.yaml` as example for the debian package [#3186](https://github.com/juanfont/headscale/pull/3186) - Fix user-owned re-registration with zero client expiry and no default storing `0001-01-01 00:00:00` in the database instead of `NULL` [#3199](https://github.com/juanfont/headscale/pull/3199) - - Pre-existing rows with `0001-01-01 00:00:00` are not backfilled; they clear themselves the next time the node re-registers - Fix `tailscaled` restart on a node with no expiry resetting `NULL` to `0001-01-01 00:00:00` in the database, affecting both tagged and untagged nodes [#3197](https://github.com/juanfont/headscale/pull/3197) +- Backfill `nodes.expiry` rows persisted by older versions as `0001-01-01 00:00:00` to `NULL`, so nodes upgraded from <0.28 stop reporting as expired [#3284](https://github.com/juanfont/headscale/issues/3284) ## 0.28.0 (2026-02-04) diff --git a/hscontrol/db/db.go b/hscontrol/db/db.go index d95b7496..8c088c49 100644 --- a/hscontrol/db/db.go +++ b/hscontrol/db/db.go @@ -722,6 +722,28 @@ WHERE tags IS NOT NULL AND tags != '[]' AND tags != ''; }, Rollback: func(db *gorm.DB) error { return nil }, }, + { + // Clear zero-time node expiry values to NULL. + // Versions before 0.28 persisted a pointer to a zero + // time.Time as '0001-01-01 00:00:00+00:00' rather than + // NULL, which 0.29 reports as an expired node. This + // normalises the existing rows so the column once + // again means "no expiry" when unset. + ID: "202605221435-clear-zero-time-node-expiry", + Migrate: func(tx *gorm.DB) error { + err := tx.Exec(` +UPDATE nodes +SET expiry = NULL +WHERE expiry IS NOT NULL AND expiry < '1900-01-01'; + `).Error + if err != nil { + return fmt.Errorf("clearing zero-time node expiry: %w", err) + } + + return nil + }, + Rollback: func(db *gorm.DB) error { return nil }, + }, }, ) diff --git a/hscontrol/db/db_test.go b/hscontrol/db/db_test.go index 20e95e7b..60abcfa0 100644 --- a/hscontrol/db/db_test.go +++ b/hscontrol/db/db_test.go @@ -144,6 +144,60 @@ func TestSQLiteMigrationAndDataValidation(t *testing.T) { assert.NotContains(t, node7.Tags, "tag:forbidden", "node7 should NOT have tag:forbidden (unauthorized)") }, }, + // Test for the zero-time node expiry migration + // (202605221435-clear-zero-time-node-expiry). Pre-0.28 versions + // stored a zero time.Time as '0001-01-01 00:00:00+00:00' rather + // than NULL, which caused 0.29 to report those nodes as expired. + // Fixes: https://github.com/juanfont/headscale/issues/3284 + { + dbPath: "testdata/sqlite/zero_time_expiry_migration_test.sql", + wantFunc: func(t *testing.T, hsdb *HSDatabase) { + t.Helper() + + nodes, err := Read(hsdb.DB, func(rx *gorm.DB) (types.Nodes, error) { + return ListNodes(rx) + }) + require.NoError(t, err) + require.Len(t, nodes, 5, "should have all 5 nodes") + + byHostname := make(map[string]*types.Node, len(nodes)) + for _, n := range nodes { + byHostname[n.Hostname] = n + } + + // Node 1 had a zero-time expiry; should be cleared. + node1 := byHostname["node1"] + require.NotNil(t, node1, "node1 should exist") + assert.Nil(t, node1.Expiry, "node1 zero-time expiry should be cleared to NULL") + assert.False(t, node1.IsExpired(), "node1 should not be reported as expired") + + // Node 2 already had NULL expiry; should still be NULL. + node2 := byHostname["node2"] + require.NotNil(t, node2, "node2 should exist") + assert.Nil(t, node2.Expiry, "node2 NULL expiry should be preserved") + assert.False(t, node2.IsExpired(), "node2 should not be reported as expired") + + // Node 3 had a real future expiry; should be preserved. + node3 := byHostname["node3"] + require.NotNil(t, node3, "node3 should exist") + require.NotNil(t, node3.Expiry, "node3 future expiry should be preserved") + assert.Equal(t, 2099, node3.Expiry.UTC().Year(), "node3 expiry year should be 2099") + assert.False(t, node3.IsExpired(), "node3 with future expiry should not be expired") + + // Node 4 had a real past expiry; should be preserved. + node4 := byHostname["node4"] + require.NotNil(t, node4, "node4 should exist") + require.NotNil(t, node4.Expiry, "node4 past expiry should be preserved") + assert.Equal(t, 2020, node4.Expiry.UTC().Year(), "node4 expiry year should be 2020") + assert.True(t, node4.IsExpired(), "node4 with past expiry should still be expired") + + // Node 5 also had a zero-time expiry; should be cleared. + node5 := byHostname["node5"] + require.NotNil(t, node5, "node5 should exist") + assert.Nil(t, node5.Expiry, "node5 zero-time expiry should be cleared to NULL") + assert.False(t, node5.IsExpired(), "node5 should not be reported as expired") + }, + }, } for _, tt := range tests { diff --git a/hscontrol/db/testdata/sqlite/zero_time_expiry_migration_test.sql b/hscontrol/db/testdata/sqlite/zero_time_expiry_migration_test.sql new file mode 100644 index 00000000..9a6d3185 --- /dev/null +++ b/hscontrol/db/testdata/sqlite/zero_time_expiry_migration_test.sql @@ -0,0 +1,82 @@ +-- Test SQL dump for zero-time node expiry migration +-- (202605221435-clear-zero-time-node-expiry) +-- +-- Pre-0.28 versions of Headscale persisted a zero time.Time as the string +-- '0001-01-01 00:00:00+00:00' in nodes.expiry instead of NULL. Upgrading +-- to 0.29 surfaces those rows as "expired" because they look like a +-- timestamp at year 1. This dump exercises the data fix. +-- Fixes: https://github.com/juanfont/headscale/issues/3284 + +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; + +-- Migrations table: all entries BEFORE the zero-time fix have been applied. +-- The new migration is intentionally absent so it runs against this dump. +CREATE TABLE `migrations` (`id` text,PRIMARY KEY (`id`)); +INSERT INTO migrations VALUES('202312101416'); +INSERT INTO migrations VALUES('202312101430'); +INSERT INTO migrations VALUES('202402151347'); +INSERT INTO migrations VALUES('2024041121742'); +INSERT INTO migrations VALUES('202406021630'); +INSERT INTO migrations VALUES('202409271400'); +INSERT INTO migrations VALUES('202407191627'); +INSERT INTO migrations VALUES('202408181235'); +INSERT INTO migrations VALUES('202501221827'); +INSERT INTO migrations VALUES('202501311657'); +INSERT INTO migrations VALUES('202502070949'); +INSERT INTO migrations VALUES('202502131714'); +INSERT INTO migrations VALUES('202502171819'); +INSERT INTO migrations VALUES('202505091439'); +INSERT INTO migrations VALUES('202505141324'); +INSERT INTO migrations VALUES('202507021200'); +INSERT INTO migrations VALUES('202510311551'); +INSERT INTO migrations VALUES('202511101554-drop-old-idx'); +INSERT INTO migrations VALUES('202511011637-preauthkey-bcrypt'); +INSERT INTO migrations VALUES('202511122344-remove-newline-index'); +INSERT INTO migrations VALUES('202511131445-node-forced-tags-to-tags'); +INSERT INTO migrations VALUES('202601121700-migrate-hostinfo-request-tags'); +INSERT INTO migrations VALUES('202602201200-clear-tagged-node-user-id'); + +-- Users table +CREATE TABLE `users` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`updated_at` datetime,`deleted_at` datetime,`name` text,`display_name` text,`email` text,`provider_identifier` text,`provider` text,`profile_pic_url` text); +INSERT INTO users VALUES(1,'2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL,'user1','User One','user1@example.com',NULL,NULL,NULL); + +-- Pre-auth keys table +CREATE TABLE `pre_auth_keys` (`id` integer PRIMARY KEY AUTOINCREMENT,`key` text,`user_id` integer,`reusable` numeric,`ephemeral` numeric DEFAULT false,`used` numeric DEFAULT false,`tags` text,`created_at` datetime,`expiration` datetime,`prefix` text,`hash` blob,CONSTRAINT `fk_pre_auth_keys_user` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON DELETE SET NULL); + +-- API keys table +CREATE TABLE `api_keys` (`id` integer PRIMARY KEY AUTOINCREMENT,`prefix` text,`hash` blob,`created_at` datetime,`expiration` datetime,`last_seen` datetime); + +-- Nodes table - current schema (after the tags rename + last_seen/expiry reordering) +CREATE TABLE IF NOT EXISTS "nodes" (`id` integer PRIMARY KEY AUTOINCREMENT,`machine_key` text,`node_key` text,`disco_key` text,`endpoints` text,`host_info` text,`ipv4` text,`ipv6` text,`hostname` text,`given_name` varchar(63),`user_id` integer,`register_method` text,`tags` text,`auth_key_id` integer,`last_seen` datetime,`expiry` datetime,`approved_routes` text,`created_at` datetime,`updated_at` datetime,`deleted_at` datetime,CONSTRAINT `fk_nodes_user` FOREIGN KEY (`user_id`) REFERENCES `users`(`id`) ON DELETE CASCADE,CONSTRAINT `fk_nodes_auth_key` FOREIGN KEY (`auth_key_id`) REFERENCES `pre_auth_keys`(`id`)); + +-- Node 1: zero-time expiry. After migration: expiry IS NULL. +INSERT INTO nodes VALUES(1,'mkey:a0ab77456320823945ae0331823e3c0d516fae9585bd42698dfa1ac3d7679e01','nodekey:7c84167ab68f494942de14deb83587fd841843de2bac105b6c670048c1605501','discokey:53075b3c6cad3b62a2a29caea61beeb93f66b8c75cb89dac465236a5bbf57701','[]','{}','100.64.0.1','fd7a:115c:a1e0::1','node1','node1',1,'cli','[]',NULL,'2024-01-01 00:00:00+00:00','0001-01-01 00:00:00+00:00','[]','2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL); + +-- Node 2: NULL expiry already. After migration: still NULL. +INSERT INTO nodes VALUES(2,'mkey:a0ab77456320823945ae0331823e3c0d516fae9585bd42698dfa1ac3d7679e02','nodekey:7c84167ab68f494942de14deb83587fd841843de2bac105b6c670048c1605502','discokey:53075b3c6cad3b62a2a29caea61beeb93f66b8c75cb89dac465236a5bbf57702','[]','{}','100.64.0.2','fd7a:115c:a1e0::2','node2','node2',1,'cli','[]',NULL,'2024-01-01 00:00:00+00:00',NULL,'[]','2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL); + +-- Node 3: real future expiry. After migration: preserved. +INSERT INTO nodes VALUES(3,'mkey:a0ab77456320823945ae0331823e3c0d516fae9585bd42698dfa1ac3d7679e03','nodekey:7c84167ab68f494942de14deb83587fd841843de2bac105b6c670048c1605503','discokey:53075b3c6cad3b62a2a29caea61beeb93f66b8c75cb89dac465236a5bbf57703','[]','{}','100.64.0.3','fd7a:115c:a1e0::3','node3','node3',1,'cli','[]',NULL,'2024-01-01 00:00:00+00:00','2099-01-01 00:00:00+00:00','[]','2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL); + +-- Node 4: real past expiry (legitimately expired). After migration: preserved. +INSERT INTO nodes VALUES(4,'mkey:a0ab77456320823945ae0331823e3c0d516fae9585bd42698dfa1ac3d7679e04','nodekey:7c84167ab68f494942de14deb83587fd841843de2bac105b6c670048c1605504','discokey:53075b3c6cad3b62a2a29caea61beeb93f66b8c75cb89dac465236a5bbf57704','[]','{}','100.64.0.4','fd7a:115c:a1e0::4','node4','node4',1,'cli','[]',NULL,'2024-01-01 00:00:00+00:00','2020-01-01 00:00:00+00:00','[]','2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL); + +-- Node 5: another zero-time row to confirm the WHERE clause matches multiple rows. +INSERT INTO nodes VALUES(5,'mkey:a0ab77456320823945ae0331823e3c0d516fae9585bd42698dfa1ac3d7679e05','nodekey:7c84167ab68f494942de14deb83587fd841843de2bac105b6c670048c1605505','discokey:53075b3c6cad3b62a2a29caea61beeb93f66b8c75cb89dac465236a5bbf57705','[]','{}','100.64.0.5','fd7a:115c:a1e0::5','node5','node5',1,'cli','[]',NULL,'2024-01-01 00:00:00+00:00','0001-01-01 00:00:00+00:00','[]','2024-01-01 00:00:00+00:00','2024-01-01 00:00:00+00:00',NULL); + +-- Policies table (empty) +CREATE TABLE `policies` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`updated_at` datetime,`deleted_at` datetime,`data` text); + +DELETE FROM sqlite_sequence; +INSERT INTO sqlite_sequence VALUES('users',1); +INSERT INTO sqlite_sequence VALUES('nodes',5); +CREATE INDEX idx_users_deleted_at ON users(deleted_at); +CREATE UNIQUE INDEX idx_api_keys_prefix ON api_keys(prefix); +CREATE INDEX idx_policies_deleted_at ON policies(deleted_at); +CREATE UNIQUE INDEX idx_provider_identifier ON users(provider_identifier) WHERE provider_identifier IS NOT NULL; +CREATE UNIQUE INDEX idx_name_provider_identifier ON users(name, provider_identifier); +CREATE UNIQUE INDEX idx_name_no_provider_identifier ON users(name) WHERE provider_identifier IS NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_pre_auth_keys_prefix ON pre_auth_keys(prefix) WHERE prefix IS NOT NULL AND prefix != ''; + +COMMIT; From 171fd7a3c54156965753a63639cdcafcd50c8d67 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Fri, 29 May 2026 10:06:02 +0000 Subject: [PATCH 05/10] policy: key autogroup:self invalidation on UserID not User view invalidateAutogroupSelfCache derived the owning user from node.User().ID(), dereferencing the User association view. The NodeStore stores nodes by value with User as a *User pointer, and not every write path hydrates that association, so a non-tagged node could reach SetNodes with UserID set but User nil. UserView.ID() then dereferenced a nil *User and panicked on /machine/map whenever an autogroup:self policy was active and a node restarted tailscaled. Group affected nodes by node.TypedUserID(), which reads the UserID field directly. UserID is the authoritative ownership field for non-tagged nodes, so this needs no hydrated association and fixes every .User().ID() site in the function. --- hscontrol/policy/v2/policy.go | 31 ++++++++------ hscontrol/policy/v2/policy_test.go | 66 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 13 deletions(-) diff --git a/hscontrol/policy/v2/policy.go b/hscontrol/policy/v2/policy.go index 7a914224..299a2e2f 100644 --- a/hscontrol/policy/v2/policy.go +++ b/hscontrol/policy/v2/policy.go @@ -1299,13 +1299,18 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S // Tagged nodes don't participate in autogroup:self (identity is tag-based), // so we skip them when collecting affected users, except when tag status changes // (which affects the user's device set). - affectedUsers := make(map[uint]struct{}) + // + // Ownership is keyed on TypedUserID (the UserID field), not the User + // association view: the NodeStore holds nodes by value with User as a + // *User pointer, and not every write path hydrates that association. A + // non-tagged node always has UserID set, so it is the reliable owner key. + affectedUsers := make(map[types.UserID]struct{}) // Check for removed nodes (only non-tagged nodes affect autogroup:self) for nodeID, oldNode := range oldNodeMap { if _, exists := newNodeMap[nodeID]; !exists { if !oldNode.IsTagged() { - affectedUsers[oldNode.User().ID()] = struct{}{} + affectedUsers[oldNode.TypedUserID()] = struct{}{} } } } @@ -1314,7 +1319,7 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S for nodeID, newNode := range newNodeMap { if _, exists := oldNodeMap[nodeID]; !exists { if !newNode.IsTagged() { - affectedUsers[newNode.User().ID()] = struct{}{} + affectedUsers[newNode.TypedUserID()] = struct{}{} } } } @@ -1327,10 +1332,10 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S if oldNode.IsTagged() != newNode.IsTagged() { if !oldNode.IsTagged() { // Was untagged, now tagged: user lost a device - affectedUsers[oldNode.User().ID()] = struct{}{} + affectedUsers[oldNode.TypedUserID()] = struct{}{} } else { // Was tagged, now untagged: user gained a device - affectedUsers[newNode.User().ID()] = struct{}{} + affectedUsers[newNode.TypedUserID()] = struct{}{} } continue @@ -1342,9 +1347,9 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S } // Check if user changed (both versions are non-tagged here) - if oldNode.User().ID() != newNode.User().ID() { - affectedUsers[oldNode.User().ID()] = struct{}{} - affectedUsers[newNode.User().ID()] = struct{}{} + if oldNode.TypedUserID() != newNode.TypedUserID() { + affectedUsers[oldNode.TypedUserID()] = struct{}{} + affectedUsers[newNode.TypedUserID()] = struct{}{} } // Check if IPs changed (simple check - could be more sophisticated) @@ -1352,12 +1357,12 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S newIPs := newNode.IPs() if len(oldIPs) != len(newIPs) { - affectedUsers[newNode.User().ID()] = struct{}{} + affectedUsers[newNode.TypedUserID()] = struct{}{} } else { // Check if any IPs are different for i, oldIP := range oldIPs { if i >= len(newIPs) || oldIP != newIPs[i] { - affectedUsers[newNode.User().ID()] = struct{}{} + affectedUsers[newNode.TypedUserID()] = struct{}{} break } } @@ -1370,7 +1375,7 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S // because autogroup:self rules depend on the entire user's device set. for nodeID := range pm.filterRulesMap { // Find the user for this cached node - var nodeUserID uint + var nodeUserID types.UserID found := false @@ -1384,7 +1389,7 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S break } - nodeUserID = node.User().ID() + nodeUserID = node.TypedUserID() found = true break @@ -1400,7 +1405,7 @@ func (pm *PolicyManager) invalidateAutogroupSelfCache(oldNodes, newNodes views.S break } - nodeUserID = node.User().ID() + nodeUserID = node.TypedUserID() found = true break diff --git a/hscontrol/policy/v2/policy_test.go b/hscontrol/policy/v2/policy_test.go index bb90afc1..ef28bf36 100644 --- a/hscontrol/policy/v2/policy_test.go +++ b/hscontrol/policy/v2/policy_test.go @@ -226,6 +226,72 @@ func TestInvalidateAutogroupSelfCache(t *testing.T) { } } +// TestSetNodesAutogroupSelfUnhydratedUser reproduces the panic seen on +// /machine/map when an autogroup:self policy is active and a non-tagged +// node reaches the policy manager with its UserID set but the User +// association left unhydrated (User pointer nil). The NodeStore stores +// nodes by value with User as a *User; not every write path hydrates the +// association, so the autogroup:self cache invalidation must derive the +// owning user from UserID, not from the User view. +func TestSetNodesAutogroupSelfUnhydratedUser(t *testing.T) { + users := types.Users{ + {Model: gorm.Model{ID: 1}, Name: "user1", Email: "user1@headscale.net"}, + {Model: gorm.Model{ID: 2}, Name: "user2", Email: "user2@headscale.net"}, + } + + policy := `{ + "acls": [ + { + "action": "accept", + "src": ["autogroup:member"], + "dst": ["autogroup:self:*"] + } + ] + }` + + // unhydratedNode mirrors a NodeStore snapshot entry whose UserID is + // set (so it is unambiguously user-owned, not tagged) but whose User + // association was never loaded. + unhydratedNode := func(name, ipv4, ipv6 string, userID uint) *types.Node { + return &types.Node{ + Hostname: name, + IPv4: ap(ipv4), + IPv6: ap(ipv6), + UserID: new(userID), + User: nil, + } + } + + initialNodes := types.Nodes{ + node("user1-node1", "100.64.0.1", "fd7a:115c:a1e0::1", users[0]), + node("user2-node1", "100.64.0.2", "fd7a:115c:a1e0::2", users[1]), + } + for i, n := range initialNodes { + n.ID = types.NodeID(i + 1) //nolint:gosec // safe conversion in test + } + + pm, err := NewPolicyManager([]byte(policy), users, initialNodes.ViewSlice()) + require.NoError(t, err) + + require.False(t, initialNodes[0].IsTagged(), "node must be user-owned for autogroup:self") + + // Simulate a node restarting tailscaled: the same node is pushed back + // into the policy manager, but the snapshot version has no hydrated + // User association. This is the exact shape that crashed beta.1. + updatedNodes := types.Nodes{ + unhydratedNode("user1-node1", "100.64.0.1", "fd7a:115c:a1e0::1", users[0].ID), + node("user2-node1", "100.64.0.2", "fd7a:115c:a1e0::2", users[1]), + } + for i, n := range updatedNodes { + n.ID = types.NodeID(i + 1) //nolint:gosec // safe conversion in test + } + + require.NotPanics(t, func() { + _, err = pm.SetNodes(updatedNodes.ViewSlice()) + }, "SetNodes must not panic when a non-tagged node has an unhydrated User") + require.NoError(t, err) +} + // TestInvalidateGlobalPolicyCache tests the cache invalidation logic for global policies. func TestInvalidateGlobalPolicyCache(t *testing.T) { mustIPPtr := func(s string) *netip.Addr { From bceac495f97f364e0f5a6c2ed67dfa44f52a68c8 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 3 Jun 2026 10:25:20 +0000 Subject: [PATCH 06/10] policy: add NodeNeedsPeerRecompute predicate Reports whether a node's online/offline transition forces peers to recompute their netmap. True for subnet routers, relay targets (tailscale.com/cap/relay), and via targets; false otherwise. The relay-target IP set and via-target tag set are precompiled from the grants in updateLocked, alongside the existing filter, so the per-node check is a cheap set lookup. Keyed on the node itself, so an ordinary node in a tailnet that uses relay or via for other nodes is still classified as not needing a recompute. Updates #3293 --- hscontrol/policy/pm.go | 8 ++ hscontrol/policy/v2/compiled.go | 52 ++++++++ hscontrol/policy/v2/node_recompute_test.go | 137 +++++++++++++++++++++ hscontrol/policy/v2/policy.go | 56 +++++++++ 4 files changed, 253 insertions(+) create mode 100644 hscontrol/policy/v2/node_recompute_test.go diff --git a/hscontrol/policy/pm.go b/hscontrol/policy/pm.go index 5709a3f5..4c9dc397 100644 --- a/hscontrol/policy/pm.go +++ b/hscontrol/policy/pm.go @@ -36,6 +36,14 @@ type PolicyManager interface { // NodeCanApproveRoute reports whether the given node can approve the given route. NodeCanApproveRoute(node types.NodeView, route netip.Prefix) bool + // NodeNeedsPeerRecompute reports whether peers must recompute their + // netmap when the node's online state changes. True for subnet + // routers, relay targets (tailscale.com/cap/relay), and via targets; + // false for ordinary nodes, which only need a lightweight online or + // offline peer patch. [State.Connect] and [State.Disconnect] use it to + // avoid a tailnet-wide recompute on every ordinary reconnect. + NodeNeedsPeerRecompute(node types.NodeView) bool + // ViaRoutesForPeer computes via grant effects for a viewer-peer pair. // It returns which routes should be included (peer is via-designated for viewer) // and excluded (steered to a different peer). When no via grants apply, diff --git a/hscontrol/policy/v2/compiled.go b/hscontrol/policy/v2/compiled.go index 6019baa6..1f8c9ad4 100644 --- a/hscontrol/policy/v2/compiled.go +++ b/hscontrol/policy/v2/compiled.go @@ -594,6 +594,58 @@ func hasPerNodeGrants(grants []compiledGrant) bool { return false } +// collectRelayTargetIPs returns the set of IPs that are destinations of a +// tailscale.com/cap/relay grant. A node whose IP is in this set is a relay +// target: when it goes offline, peers holding a PeerRelay allocation through +// it must recompute their netmap to drop the now-dead allocation. The relay +// cap is carried on each grant's [tailcfg.CapGrant] with Dsts set to the +// resolved relay destinations (see [Policy.compileOtherDests]); the reversed +// companion rule carries [tailcfg.PeerCapabilityRelayTarget] instead and is +// intentionally skipped. +func collectRelayTargetIPs(grants []compiledGrant) (*netipx.IPSet, error) { + var b netipx.IPSetBuilder + + for i := range grants { + for _, rule := range grants[i].rules { + for _, cg := range rule.CapGrant { + if _, ok := cg.CapMap[tailcfg.PeerCapabilityRelay]; !ok { + continue + } + + for _, dst := range cg.Dsts { + b.AddPrefix(dst) + } + } + } + } + + return b.IPSet() +} + +// collectViaTargetTags returns the set of tags used as via targets across all +// grants. A node carrying any of these tags is a via target: peers steering +// traffic through it must recompute when it goes offline. Returns nil when no +// via grants exist. +func collectViaTargetTags(grants []compiledGrant) map[Tag]struct{} { + var tags map[Tag]struct{} + + for i := range grants { + if grants[i].via == nil { + continue + } + + for _, t := range grants[i].via.viaTags { + if tags == nil { + tags = make(map[Tag]struct{}) + } + + tags[t] = struct{}{} + } + } + + return tags +} + // globalFilterRules extracts global filter rules from [compiledGrant]s. // Via grants produce no global rules (they are per-node only); regular // grants contribute their full pre-compiled ruleset; self grants diff --git a/hscontrol/policy/v2/node_recompute_test.go b/hscontrol/policy/v2/node_recompute_test.go new file mode 100644 index 00000000..ed8c9db1 --- /dev/null +++ b/hscontrol/policy/v2/node_recompute_test.go @@ -0,0 +1,137 @@ +package v2 + +import ( + "net/netip" + "testing" + + "github.com/juanfont/headscale/hscontrol/types" + "github.com/stretchr/testify/require" + "gorm.io/gorm" + "tailscale.com/tailcfg" +) + +// TestNodeNeedsPeerRecompute pins which node roles force peers to recompute +// their netmap when the node's online state changes. An ordinary node only +// needs the lightweight online/offline peer patch; subnet routers, relay +// targets, and via targets change what peers compute and therefore need a +// full recompute. The predicate is keyed on the flipping node, so an ordinary +// node in a tailnet that uses relay or via elsewhere must still be classified +// as not needing a recompute. +func TestNodeNeedsPeerRecompute(t *testing.T) { + users := types.Users{ + {Model: gorm.Model{ID: 1}, Name: "user1", Email: "user1@headscale.net"}, + } + + const allowAll = `{"acls":[{"action":"accept","src":["*"],"dst":["*:*"]}]}` + + relayPol := `{ + "tagOwners": {"tag:relay": ["user1@"]}, + "grants": [ + {"src": ["*"], "dst": ["tag:relay"], "app": {"tailscale.com/cap/relay": [{}]}} + ] + }` + + viaPol := `{ + "tagOwners": {"tag:via": ["user1@"]}, + "grants": [ + {"src": ["*"], "dst": ["10.0.0.0/24"], "ip": ["*"], "via": ["tag:via"]} + ] + }` + + taildrivePol := `{ + "tagOwners": {"tag:drive": ["user1@"]}, + "grants": [ + {"src": ["*"], "dst": ["tag:drive"], "app": {"tailscale.com/cap/drive": [{}]}} + ] + }` + + ordinary := node("ordinary", "100.64.0.1", "fd7a:115c:a1e0::1", users[0]) + ordinary.ID = 1 + + subnetRouter := node("subnet", "100.64.0.2", "fd7a:115c:a1e0::2", users[0]) + subnetRouter.ID = 2 + subnetRouter.Hostinfo = &tailcfg.Hostinfo{ + RoutableIPs: []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, + } + subnetRouter.ApprovedRoutes = []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")} + + relayTarget := node("relay", "100.64.0.3", "fd7a:115c:a1e0::3", users[0]) + relayTarget.ID = 3 + relayTarget.Tags = []string{"tag:relay"} + + viaTarget := node("via", "100.64.0.4", "fd7a:115c:a1e0::4", users[0]) + viaTarget.ID = 4 + viaTarget.Tags = []string{"tag:via"} + + driveTarget := node("drive", "100.64.0.5", "fd7a:115c:a1e0::5", users[0]) + driveTarget.ID = 5 + driveTarget.Tags = []string{"tag:drive"} + + tests := []struct { + name string + pol string + nodes types.Nodes + subject *types.Node + want bool + }{ + { + name: "ordinary node under allow-all does not need recompute", + pol: allowAll, + nodes: types.Nodes{ordinary}, + subject: ordinary, + want: false, + }, + { + name: "subnet router needs recompute", + pol: allowAll, + nodes: types.Nodes{subnetRouter}, + subject: subnetRouter, + want: true, + }, + { + name: "relay target needs recompute", + pol: relayPol, + nodes: types.Nodes{relayTarget, ordinary}, + subject: relayTarget, + want: true, + }, + { + name: "ordinary node in a relay-using tailnet does not need recompute", + pol: relayPol, + nodes: types.Nodes{relayTarget, ordinary}, + subject: ordinary, + want: false, + }, + { + name: "via target needs recompute", + pol: viaPol, + nodes: types.Nodes{viaTarget, ordinary}, + subject: viaTarget, + want: true, + }, + { + name: "ordinary node in a via-using tailnet does not need recompute", + pol: viaPol, + nodes: types.Nodes{viaTarget, ordinary}, + subject: ordinary, + want: false, + }, + { + name: "taildrive target does not need recompute", + pol: taildrivePol, + nodes: types.Nodes{driveTarget, ordinary}, + subject: driveTarget, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pm, err := NewPolicyManager([]byte(tt.pol), users, tt.nodes.ViewSlice()) + require.NoError(t, err) + + got := pm.NodeNeedsPeerRecompute(tt.subject.View()) + require.Equal(t, tt.want, got) + }) + } +} diff --git a/hscontrol/policy/v2/policy.go b/hscontrol/policy/v2/policy.go index 299a2e2f..d1c570ad 100644 --- a/hscontrol/policy/v2/policy.go +++ b/hscontrol/policy/v2/policy.go @@ -45,6 +45,15 @@ type PolicyManager struct { autoApproveMapHash deephash.Sum autoApproveMap map[netip.Prefix]*netipx.IPSet + // relayTargetIPs holds the IPs of nodes that are destinations of a + // tailscale.com/cap/relay grant; viaTargetTags holds the tags used as + // via targets. A node matching either, or that is a subnet router, + // forces peers to recompute their netmap when its online state changes + // (see [PolicyManager.NodeNeedsPeerRecompute]). Recomputed from the + // compiled grants on every policy/user/node change. + relayTargetIPs *netipx.IPSet + viaTargetTags map[Tag]struct{} + // Lazy map of SSH policies sshPolicyMap map[types.NodeID]*tailcfg.SSHPolicy @@ -223,6 +232,14 @@ func (pm *PolicyManager) updateLocked() (bool, error) { pm.compiledGrants = pm.pol.compileGrants(pm.users, pm.nodes) pm.userNodeIdx = buildUserNodeIndex(pm.nodes) pm.needsPerNodeFilter = hasPerNodeGrants(pm.compiledGrants) + pm.viaTargetTags = collectViaTargetTags(pm.compiledGrants) + + relayTargetIPs, err := collectRelayTargetIPs(pm.compiledGrants) + if err != nil { + return false, fmt.Errorf("collecting relay target IPs: %w", err) + } + + pm.relayTargetIPs = relayTargetIPs var filter []tailcfg.FilterRule if pm.pol == nil || (pm.pol.ACLs == nil && pm.pol.Grants == nil) { @@ -360,6 +377,45 @@ func (pm *PolicyManager) updateLocked() (bool, error) { return true, nil } +// NodeNeedsPeerRecompute reports whether peers must recompute their netmap +// when node's online state changes. A plain node only needs the lightweight +// online/offline peer patch; these roles change what peers compute when the +// node goes up or down, so they require a full recompute: +// - subnet router: primary-route failover changes peers' AllowedIPs +// - relay target (tailscale.com/cap/relay): peers must drop a stale +// PeerRelay allocation +// - via target: peers steer traffic through this node +// +// The check is keyed on the node itself, so an ordinary node in a tailnet +// that uses relay or via for other nodes is correctly classified as not +// needing a recompute. +func (pm *PolicyManager) NodeNeedsPeerRecompute(node types.NodeView) bool { + if !node.Valid() { + return false + } + + // Subnet-router status is intrinsic to the node, so it needs no policy + // state and is checked without the lock. + if node.IsSubnetRouter() { + return true + } + + pm.mu.Lock() + defer pm.mu.Unlock() + + if pm.relayTargetIPs != nil && node.InIPSet(pm.relayTargetIPs) { + return true + } + + for tag := range pm.viaTargetTags { + if node.HasTag(string(tag)) { + return true + } + } + + return false +} + // SSHPolicy returns the [tailcfg.SSHPolicy] for node, compiling and // caching on first access. Rules use SessionDuration = 0 (no // auto-approval) and emit check URLs of the form From 7706552c99b7469a3851cd435052413af5ba1bad Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 3 Jun 2026 10:25:52 +0000 Subject: [PATCH 07/10] state: gate reconnect PolicyChange on NodeNeedsPeerRecompute MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Connect and Disconnect appended change.PolicyChange() on every reconnect. PolicyChange sets RequiresRuntimePeerComputation, so the batcher rebuilt a full netmap (packet filters, SSH policy, peer serialization) for every connected node — O(N) per reconnect, O(N^2) on a restart storm. On a small VM this saturated CPU after the v0.28 -> v0.29 upgrade. Emit it only when the node's online state changes what peers compute: subnet routers, relay targets, and via targets. An ordinary reconnect now sends just the lightweight online/offline peer patch. Relay and via targets still recompute, so peers drop a stale PeerRelay allocation when a relay goes offline. Fixes #3293 --- hscontrol/state/connect_test.go | 168 ++++++++++++++++++++++++++++++++ hscontrol/state/state.go | 27 +++-- 2 files changed, 185 insertions(+), 10 deletions(-) create mode 100644 hscontrol/state/connect_test.go diff --git a/hscontrol/state/connect_test.go b/hscontrol/state/connect_test.go new file mode 100644 index 00000000..b1ded3ab --- /dev/null +++ b/hscontrol/state/connect_test.go @@ -0,0 +1,168 @@ +package state + +import ( + "net/netip" + "testing" + + "github.com/juanfont/headscale/hscontrol/types" + "github.com/juanfont/headscale/hscontrol/types/change" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "tailscale.com/tailcfg" +) + +// runtimePeerComputationReasons returns the Reason of every change in cs that +// carries RequiresRuntimePeerComputation. Each such change makes the batcher +// rebuild a full netmap (packet filters, SSH policy, peer serialization) for +// every connected node, so it is the expensive fan-out the issue is about. +func runtimePeerComputationReasons(cs []change.Change) []string { + var reasons []string + + for _, c := range cs { + if c.RequiresRuntimePeerComputation { + reasons = append(reasons, c.Reason) + } + } + + return reasons +} + +// hasPeerPatch reports whether any change carries a lightweight PeerChange +// patch (e.g. the online/offline indicator). This is the cheap notification a +// reconnect should produce instead of a full runtime recompute. +func hasPeerPatch(cs []change.Change) bool { + for _, c := range cs { + if len(c.PeerPatches) > 0 { + return true + } + } + + return false +} + +// forcesPeerRecompute reports whether any change makes peers rebuild a full +// netmap, whether via a full update (subnet-router path) or a runtime peer +// computation (relay/via path). +func forcesPeerRecompute(cs []change.Change) bool { + for _, c := range cs { + if c.IsFull() || c.RequiresRuntimePeerComputation { + return true + } + } + + return false +} + +// TestConnectDisconnectOrdinaryNodeNoRuntimeRecompute asserts that an ordinary +// node coming online or going offline only sends the lightweight online/offline +// peer patch and does not trigger a runtime peer recompute. +// +// State.Connect and State.Disconnect gate change.PolicyChange() (which sets +// RequiresRuntimePeerComputation, forcing the batcher to rebuild a full netmap +// for every connected node) on NodeNeedsPeerRecompute. An ordinary node is +// neither a subnet router, a relay target, nor a via target, so the gate is +// false and no recompute is emitted. +// +// Emitting that recompute unconditionally turned each reconnect into O(N) full +// netmap rebuilds (and a reconnect storm into O(N^2)), which saturated CPU +// after the v0.28 -> v0.29 upgrade. +func TestConnectDisconnectOrdinaryNodeNoRuntimeRecompute(t *testing.T) { + _, s, nodeID := persistTestSetup(t) + t.Cleanup(func() { _ = s.Close() }) + + t.Run("connect", func(t *testing.T) { + cs, epoch := s.Connect(nodeID) + require.NotZero(t, epoch, "Connect should return a session epoch") + + assert.True(t, hasPeerPatch(cs), + "Connect should still emit a lightweight online peer patch") + + reasons := runtimePeerComputationReasons(cs) + assert.Empty(t, reasons, + "ordinary node connect must not trigger a runtime peer recompute; "+ + "got RequiresRuntimePeerComputation changes: %v", reasons) + }) + + t.Run("disconnect", func(t *testing.T) { + // Connect first so Disconnect's epoch check passes. + _, epoch := s.Connect(nodeID) + + cs, err := s.Disconnect(nodeID, epoch) + require.NoError(t, err) + + assert.True(t, hasPeerPatch(cs), + "Disconnect should still emit a lightweight offline peer patch") + + reasons := runtimePeerComputationReasons(cs) + assert.Empty(t, reasons, + "ordinary node disconnect must not trigger a runtime peer recompute; "+ + "got RequiresRuntimePeerComputation changes: %v", reasons) + }) +} + +// TestConnectDisconnectRelayTargetTriggersRecompute locks the cap/relay case: +// a relay target is not a subnet router, so the only thing that makes its +// connect/disconnect emit a runtime peer recompute is the +// NodeNeedsPeerRecompute gate. Peers must still receive that recompute so they +// drop a stale PeerRelay allocation when the relay goes offline. +func TestConnectDisconnectRelayTargetTriggersRecompute(t *testing.T) { + _, s, nodeID := persistTestSetup(t) + t.Cleanup(func() { _ = s.Close() }) + + // A cap/relay grant whose destination resolves to the node's owning + // user makes the node a relay target without making it a subnet router. + relayPolicy := `{"grants":[{"src":["*"],"dst":["persist-user@"],"app":{"tailscale.com/cap/relay":[{}]}}]}` + _, err := s.SetPolicy([]byte(relayPolicy)) + require.NoError(t, err) + + t.Run("connect", func(t *testing.T) { + cs, epoch := s.Connect(nodeID) + require.NotZero(t, epoch) + + assert.NotEmpty(t, runtimePeerComputationReasons(cs), + "relay-target node connect must trigger a runtime peer recompute") + }) + + t.Run("disconnect", func(t *testing.T) { + _, epoch := s.Connect(nodeID) + + cs, err := s.Disconnect(nodeID, epoch) + require.NoError(t, err) + + assert.NotEmpty(t, runtimePeerComputationReasons(cs), + "relay-target node disconnect must trigger a runtime peer recompute") + }) +} + +// TestConnectDisconnectSubnetRouterForcesRecompute guards that a subnet router +// still forces peers to recompute on connect/disconnect (primary-route +// failover changes their AllowedIPs), so the gate does not over-suppress. +func TestConnectDisconnectSubnetRouterForcesRecompute(t *testing.T) { + _, s, nodeID := persistTestSetup(t) + t.Cleanup(func() { _ = s.Close() }) + + route := netip.MustParsePrefix("10.0.0.0/24") + _, ok := s.nodeStore.UpdateNode(nodeID, func(n *types.Node) { + n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: []netip.Prefix{route}} + n.ApprovedRoutes = []netip.Prefix{route} + }) + require.True(t, ok) + + t.Run("connect", func(t *testing.T) { + cs, epoch := s.Connect(nodeID) + require.NotZero(t, epoch) + + assert.True(t, forcesPeerRecompute(cs), + "subnet router connect must force a peer recompute") + }) + + t.Run("disconnect", func(t *testing.T) { + _, epoch := s.Connect(nodeID) + + cs, err := s.Disconnect(nodeID, epoch) + require.NoError(t, err) + + assert.True(t, forcesPeerRecompute(cs), + "subnet router disconnect must force a peer recompute") + }) +} diff --git a/hscontrol/state/state.go b/hscontrol/state/state.go index dce769b7..4daf18f3 100644 --- a/hscontrol/state/state.go +++ b/hscontrol/state/state.go @@ -594,9 +594,14 @@ func (s *State) Connect(id types.NodeID) ([]change.Change, uint64) { c = append(c, change.NodeAdded(id)) } - // Coming online may re-enable cap/relay grants and identity-based - // aliases targeting this node, so peers need a fresh netmap. - c = append(c, change.PolicyChange()) + // Only a node whose online state changes what peers compute (a subnet + // router, relay target, or via target) needs a full peer recompute. + // An ordinary node coming online just sends the lightweight online + // patch above; emitting a PolicyChange for it would force every peer + // to rebuild its netmap on every reconnect. + if s.polMan.NodeNeedsPeerRecompute(node) { + c = append(c, change.PolicyChange()) + } return c, epoch } @@ -648,13 +653,15 @@ func (s *State) Disconnect(id types.NodeID, epoch uint64) ([]change.Change, erro c = change.Change{} } - // Going offline can affect policy compilation beyond subnet routes - // (cap/relay grants, tag/group aliases, via routes), so peers need - // a fresh netmap regardless of whether the primary moved. - // - // TODO(kradalby): fires one full netmap recompute per peer on - // every connect/disconnect. Coalesce in mapper/batcher.go:addToBatch. - cs := []change.Change{change.NodeOfflineFor(node), c, change.PolicyChange()} + // Only a node whose online state changes what peers compute (a subnet + // router, relay target, or via target) needs a full peer recompute. + // An ordinary node going offline just sends the lightweight offline + // patch; emitting a PolicyChange for it would force every peer to + // rebuild its netmap on every disconnect. + cs := []change.Change{change.NodeOfflineFor(node), c} + if s.polMan.NodeNeedsPeerRecompute(node) { + cs = append(cs, change.PolicyChange()) + } return cs, nil } From cffdb77c8ba4ba3e61230557fefd376f131d94bc Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 3 Jun 2026 13:21:59 +0000 Subject: [PATCH 08/10] mapper, change: coalesce duplicate policy recomputes per tick Deduplicate broadcast policy changes within a tick so peers don't recompute their netmap more than needed during a reconnect storm. Updates #3293 --- hscontrol/mapper/batcher.go | 3 + hscontrol/mapper/batcher_test.go | 66 +++++++++++++++++++ hscontrol/types/change/change.go | 35 ++++++++++ hscontrol/types/change/change_test.go | 93 +++++++++++++++++++++++++++ 4 files changed, 197 insertions(+) diff --git a/hscontrol/mapper/batcher.go b/hscontrol/mapper/batcher.go index 66b7b582..005dca8a 100644 --- a/hscontrol/mapper/batcher.go +++ b/hscontrol/mapper/batcher.go @@ -630,6 +630,9 @@ func (b *Batcher) processBatchedChanges() { return true } + // One policy recompute rebuilds the whole netmap; drop same-tick repeats. + pending = change.DedupePolicyChanges(pending) + // Queue a single work item containing all pending changes. // One item per node ensures a single worker processes them // sequentially, preventing out-of-order delivery. diff --git a/hscontrol/mapper/batcher_test.go b/hscontrol/mapper/batcher_test.go index 738c0b6a..49aeb54d 100644 --- a/hscontrol/mapper/batcher_test.go +++ b/hscontrol/mapper/batcher_test.go @@ -1104,6 +1104,72 @@ func TestBatcherWorkQueueBatching(t *testing.T) { } } +// TestBatcherCoalescesPolicyRecomputesPerTick proves that many identical +// broadcast policy changes arriving in a single batcher tick collapse to one +// runtime peer recompute per node. Without coalescing, each PolicyChange drives +// a separate full netmap rebuild for every connected node (the reconnect-storm +// fan-out); with it, a node sees at most one recompute per tick. +func TestBatcherCoalescesPolicyRecomputesPerTick(t *testing.T) { + for _, bf := range allBatcherFunctions { + t.Run(bf.name, func(t *testing.T) { + const ( + nodesPerUser = 4 + policyChangesPerTick = 8 + ) + + testData, cleanup := setupBatcherWithTestData(t, bf.fn, 1, nodesPerUser, 100) + defer cleanup() + + batcher := testData.Batcher + for i := range testData.Nodes { + n := &testData.Nodes[i] + require.NoError(t, batcher.AddNode(n.n.ID, n.ch, tailcfg.CapabilityVersion(100), nil)) + } + + // Many identical broadcast policy changes, then a DERP-map change as + // a sentinel. All land in one tick; the sentinel rides the same work + // item after the recompute(s), so its arrival marks the end of this + // tick's policy responses for a node. + for range policyChangesPerTick { + batcher.AddWork(change.PolicyChange()) + } + + batcher.AddWork(change.DERPMap()) + + for i := range testData.Nodes { + id := testData.Nodes[i].n.ID + ch := testData.Nodes[i].ch + + policyResponses := 0 + deadline := time.After(2 * time.Second) + + drain: + for { + select { + case resp := <-ch: + switch { + case resp.DERPMap != nil && len(resp.Peers) == 0: + // Sentinel: every policy recompute for this tick has + // already been delivered to this node. + break drain + case len(resp.PacketFilters) > 0 && len(resp.Peers) == 0: + // A runtime peer recompute (policyChangeResponse): + // packet filters and incremental peers, no full list. + policyResponses++ + } + case <-deadline: + t.Fatalf("node %d never received the DERP sentinel", id) + } + } + + assert.LessOrEqualf(t, policyResponses, 1, + "node %d received %d policy recomputes in one tick; identical recomputes must coalesce to one", + id, policyResponses) + } + }) + } +} + // TestBatcherWorkerChannelSafety tests that worker goroutines handle closed // channels safely without panicking when processing work items. // diff --git a/hscontrol/types/change/change.go b/hscontrol/types/change/change.go index abfcaee2..5b3c933f 100644 --- a/hscontrol/types/change/change.go +++ b/hscontrol/types/change/change.go @@ -250,6 +250,41 @@ func FilterForNode(nodeID types.NodeID, rs []Change) []Change { return result } +// IsBroadcastPolicyChange reports whether r is a tailnet-wide policy recompute +// with no per-node payload. A recompute reads the current snapshot, so every +// such change is interchangeable and same-tick duplicates are redundant. A +// targeted or self-update ([Change.OriginNode]) recompute is per-node, so it is +// not one of these. +func (r Change) IsBroadcastPolicyChange() bool { + return r.RequiresRuntimePeerComputation && !r.IsTargetedToNode() && r.OriginNode == 0 +} + +// DedupePolicyChanges keeps the first broadcast policy change in a tick and +// drops the rest: each rebuilds a node's whole netmap from the same snapshot, so +// the repeats are wasted work. Order and all other changes are preserved. +func DedupePolicyChanges(changes []Change) []Change { + if len(changes) < 2 { + return changes + } + + out := make([]Change, 0, len(changes)) + seen := false + + for _, r := range changes { + if r.IsBroadcastPolicyChange() { + if seen { + continue + } + + seen = true + } + + out = append(out, r) + } + + return out +} + func uniqueNodeIDs(ids []types.NodeID) []types.NodeID { if len(ids) == 0 { return nil diff --git a/hscontrol/types/change/change_test.go b/hscontrol/types/change/change_test.go index 729dc468..c5086267 100644 --- a/hscontrol/types/change/change_test.go +++ b/hscontrol/types/change/change_test.go @@ -296,6 +296,99 @@ func TestChange_Merge(t *testing.T) { } } +func TestChange_IsBroadcastPolicyChange(t *testing.T) { + originUpdate := PolicyChange() + originUpdate.OriginNode = 7 + + targeted := PolicyChange() + targeted.TargetNode = 7 + + tests := []struct { + name string + c Change + want bool + }{ + {name: "policy change", c: PolicyChange(), want: true}, + {name: "self-update recompute", c: originUpdate, want: false}, + {name: "targeted recompute", c: targeted, want: false}, + {name: "online patch", c: NodeOnline(1), want: false}, + {name: "full update", c: FullUpdate(), want: false}, + {name: "derp map", c: DERPMap(), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, tt.c.IsBroadcastPolicyChange()) + }) + } +} + +func TestDedupePolicyChanges(t *testing.T) { + // originRecompute is a runtime recompute carrying node-specific payload + // (OriginNode), so it is not the canonical broadcast PolicyChange and must + // never be coalesced away. + originRecompute := PolicyChange() + originRecompute.OriginNode = 7 + + tests := []struct { + name string + changes []Change + want []Change + }{ + { + name: "nil is a no-op", + changes: nil, + want: nil, + }, + { + name: "single policy change is unchanged", + changes: []Change{PolicyChange()}, + want: []Change{PolicyChange()}, + }, + { + name: "identical policy changes collapse to one", + changes: []Change{PolicyChange(), PolicyChange(), PolicyChange()}, + want: []Change{PolicyChange()}, + }, + { + name: "peer patches survive between collapsed policy changes", + changes: []Change{ + NodeOnline(1), PolicyChange(), NodeOnline(2), PolicyChange(), NodeOffline(3), + }, + want: []Change{ + NodeOnline(1), PolicyChange(), NodeOnline(2), NodeOffline(3), + }, + }, + { + name: "NodeAdded is preserved, not treated as a recompute", + changes: []Change{PolicyChange(), NodeAdded(5), PolicyChange()}, + want: []Change{PolicyChange(), NodeAdded(5)}, + }, + { + name: "recompute carrying OriginNode is kept alongside the canonical one", + changes: []Change{PolicyChange(), originRecompute, PolicyChange()}, + want: []Change{PolicyChange(), originRecompute}, + }, + { + name: "non-canonical recomputes are not collapsed", + changes: []Change{originRecompute, originRecompute}, + want: []Change{originRecompute, originRecompute}, + }, + { + name: "changes without any recompute are unchanged", + changes: []Change{NodeOnline(1), DERPMap()}, + want: []Change{NodeOnline(1), DERPMap()}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := DedupePolicyChanges(tt.changes) + assert.Equal(t, tt.want, got) + }) + } +} + func TestChange_Constructors(t *testing.T) { tests := []struct { name string From 5228cb1a408e29b67d72720894c80ac489f11daa Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 3 Jun 2026 13:55:42 +0000 Subject: [PATCH 09/10] change: drop subnet-router full update, use policy change Don't send a full update when a subnet router goes up or down; the gated policy change already recomputes peers and is a smaller payload. Updates #3293 --- hscontrol/state/connect_test.go | 48 +++++++++++++++++++++++++++ hscontrol/types/change/change.go | 26 +++++---------- hscontrol/types/change/change_test.go | 41 +++++++++++++++++++++++ 3 files changed, 97 insertions(+), 18 deletions(-) diff --git a/hscontrol/state/connect_test.go b/hscontrol/state/connect_test.go index b1ded3ab..3a60d1b3 100644 --- a/hscontrol/state/connect_test.go +++ b/hscontrol/state/connect_test.go @@ -166,3 +166,51 @@ func TestConnectDisconnectSubnetRouterForcesRecompute(t *testing.T) { "subnet router disconnect must force a peer recompute") }) } + +// TestConnectDisconnectSubnetRouterEmitsPolicyChangeNotFull pins how a subnet +// router forces that recompute: through the gated change.PolicyChange() (a +// runtime peer recompute) and the lightweight online/offline peer patch, not a +// full update. policyChangeResponse is a strict subset of a full update yet +// still carries primary-route failover, so the heavier FullUpdate that the +// online/offline change once emitted for subnet routers is unnecessary. +func TestConnectDisconnectSubnetRouterEmitsPolicyChangeNotFull(t *testing.T) { + _, s, nodeID := persistTestSetup(t) + t.Cleanup(func() { _ = s.Close() }) + + route := netip.MustParsePrefix("10.0.0.0/24") + _, ok := s.nodeStore.UpdateNode(nodeID, func(n *types.Node) { + n.Hostinfo = &tailcfg.Hostinfo{RoutableIPs: []netip.Prefix{route}} + n.ApprovedRoutes = []netip.Prefix{route} + }) + require.True(t, ok) + + assertRecomputeNotFull := func(t *testing.T, cs []change.Change) { + t.Helper() + + assert.NotEmpty(t, runtimePeerComputationReasons(cs), + "subnet router must still drive a runtime peer recompute") + assert.True(t, hasPeerPatch(cs), + "subnet router should still emit the lightweight online/offline patch") + + for _, c := range cs { + assert.Falsef(t, c.IsFull(), + "subnet router recompute must be a PolicyChange, not a full update: %q", c.Reason) + } + } + + t.Run("connect", func(t *testing.T) { + cs, epoch := s.Connect(nodeID) + require.NotZero(t, epoch) + + assertRecomputeNotFull(t, cs) + }) + + t.Run("disconnect", func(t *testing.T) { + _, epoch := s.Connect(nodeID) + + cs, err := s.Disconnect(nodeID, epoch) + require.NoError(t, err) + + assertRecomputeNotFull(t, cs) + }) +} diff --git a/hscontrol/types/change/change.go b/hscontrol/types/change/change.go index 5b3c933f..77d0f083 100644 --- a/hscontrol/types/change/change.go +++ b/hscontrol/types/change/change.go @@ -456,29 +456,19 @@ func NodeRemoved(id types.NodeID) Change { return PeersRemoved(id) } -// NodeOnlineFor returns a [Change] for when a node comes online. -// If the node is a subnet router, a full update is sent instead of a patch. +// NodeOnlineFor returns the [Change] for a node coming online: a lightweight +// [NodeOnline] peer patch. Subnet routers, relay targets, and via targets get +// their full peer recompute from the gated [PolicyChange] that State.Connect +// emits, so no full update is needed here. func NodeOnlineFor(node types.NodeView) Change { - if node.IsSubnetRouter() { - c := FullUpdate() - c.Reason = "subnet router online" - - return c - } - return NodeOnline(node.ID()) } -// NodeOfflineFor returns a [Change] for when a node goes offline. -// If the node is a subnet router, a full update is sent instead of a patch. +// NodeOfflineFor returns the [Change] for a node going offline: a lightweight +// [NodeOffline] peer patch. As with [NodeOnlineFor], subnet routers and other +// recompute-forcing nodes rely on the gated [PolicyChange] from State.Disconnect +// for the peer recompute, so no full update is needed here. func NodeOfflineFor(node types.NodeView) Change { - if node.IsSubnetRouter() { - c := FullUpdate() - c.Reason = "subnet router offline" - - return c - } - return NodeOffline(node.ID()) } diff --git a/hscontrol/types/change/change_test.go b/hscontrol/types/change/change_test.go index c5086267..14fe077a 100644 --- a/hscontrol/types/change/change_test.go +++ b/hscontrol/types/change/change_test.go @@ -1,11 +1,13 @@ package change import ( + "net/netip" "reflect" "testing" "github.com/juanfont/headscale/hscontrol/types" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "tailscale.com/tailcfg" ) @@ -612,3 +614,42 @@ func TestUniqueNodeIDs(t *testing.T) { }) } } + +func TestNodeOnlineOfflineForSubnetRouter(t *testing.T) { + route := netip.MustParsePrefix("10.0.0.0/24") + router := types.Node{ + ID: 1, + Hostinfo: &tailcfg.Hostinfo{RoutableIPs: []netip.Prefix{route}}, + ApprovedRoutes: []netip.Prefix{route}, + } + view := router.View() + require.True(t, view.IsSubnetRouter(), "test node must be a subnet router") + + tests := []struct { + name string + got Change + wantOnline bool + }{ + {name: "online", got: NodeOnlineFor(view), wantOnline: true}, + {name: "offline", got: NodeOfflineFor(view), wantOnline: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // A subnet router's online/offline transition rides the lightweight + // peer patch, not a full update: the gated PolicyChange that + // State.Connect/Disconnect emit owns the netmap recompute. + assert.False(t, tt.got.IsFull(), + "subnet router online/offline must be a peer patch, not a full update") + + require.NotEmpty(t, tt.got.PeerPatches, + "expected an online/offline peer patch") + + patch := tt.got.PeerPatches[0] + assert.Equal(t, view.ID().NodeID(), patch.NodeID) + + require.NotNil(t, patch.Online) + assert.Equal(t, tt.wantOnline, *patch.Online) + }) + } +} From 209ba5c4ea54ec483a8246a8f0204eb0d4890662 Mon Sep 17 00:00:00 2001 From: Ryan Malloy Date: Thu, 4 Jun 2026 01:57:50 -0600 Subject: [PATCH 10/10] oidc groups: store and expose the OIDC groups claim Add a Groups column on the User model populated from the OIDC 'groups' claim at login, and surface it through the gRPC/REST User message so external tools (Headplane) can read group membership without reaching into the database. - proto: add 'repeated string groups = 9' to v1.User. - types.User: Groups text column, GetGroups/SetGroups JSON helpers, FromClaim populates from claims.Groups. The existing FlexibleStringSlice on OIDCClaims.Groups already handles JumpCloud-style single-string emission. - types.User.Proto(): populate v1.User.Groups via GetGroups(). - db migration 202505141323: add the column BEFORE the existing 202505141324 migration that loads users via the struct, so the schema is in place before any migration touches the User type. - db migration 202507021200: extend the inline CREATE TABLE users and INSERT INTO users ... SELECT FROM users_old to carry the new column through the SQLite schema-recreation step. - schema.sql: declare the column so squibble.Validate accepts databases produced by the new migration chain. Verified against all 7 historical sqlite dumps in hscontrol/db/testdata/sqlite. - types.UserView, types_clone.go: regenerated to expose Groups. - config-example.yaml, docs/ref/oidc.md: note the 'groups' scope and the role the column plays for external integrations. - integration/oidc_groups_test.go: verify the round-trip via headscale.ListUsers() for users with multi-group, single-group, and empty group memberships. --- config-example.yaml | 3 +- docs/ref/oidc.md | 55 ++++++++- gen/go/headscale/v1/user.pb.go | 16 ++- .../headscale/v1/headscale.swagger.json | 7 ++ hscontrol/db/db.go | 27 ++++- hscontrol/db/schema.sql | 1 + hscontrol/types/types_clone.go | 1 + hscontrol/types/types_view.go | 3 + hscontrol/types/users.go | 42 +++++++ hscontrol/types/users_test.go | 1 + integration/oidc_groups_test.go | 113 ++++++++++++++++++ proto/headscale/v1/user.proto | 4 + 12 files changed, 266 insertions(+), 7 deletions(-) create mode 100644 integration/oidc_groups_test.go diff --git a/config-example.yaml b/config-example.yaml index 22b9f349..04178086 100644 --- a/config-example.yaml +++ b/config-example.yaml @@ -406,9 +406,10 @@ unix_socket_permission: "0770" # use_expiry_from_token: false # # # The OIDC scopes to use, defaults to "openid", "profile" and "email". +# # Add "groups" scope to enable group storage for external integrations. # # Custom scopes can be configured as needed, be sure to always include the # # required "openid" scope. -# scope: ["openid", "profile", "email"] +# scope: ["openid", "profile", "email", "groups"] # # # Only verified email addresses are synchronized to the user profile by # # default. Unverified emails may be allowed in case an identity provider diff --git a/docs/ref/oidc.md b/docs/ref/oidc.md index 3d2e8764..ab55e28f 100644 --- a/docs/ref/oidc.md +++ b/docs/ref/oidc.md @@ -240,13 +240,64 @@ endpoint. | username | `preferred_username` | Depends on identity provider, eg: `ssmith`, `ssmith@idp.example.com`, `\\example.com\ssmith` | | profile picture | `picture` | URL to a profile picture or avatar | | provider identifier | `iss`, `sub` | A stable and unique identifier for a user, typically a combination of `iss` and `sub` OIDC claims | -| | `groups` | [Only used to filter for allowed groups](#authorize-users-with-filters) | +| group membership | `groups` | Used for [access filtering](#authorize-users-with-filters) and stored for external integrations | + +## Group Storage and Integration + +Starting with Headscale v0.24.0, OIDC group membership is automatically extracted from authentication claims and stored in the database. This enables external integrations (such as web interfaces) to implement role-based access control based on a user's group membership. + +### Group Storage +- Groups are extracted from both ID tokens and UserInfo endpoint responses +- Group membership is updated on every successful OIDC login +- Groups are stored as JSON in the user database for external access +- Multiple group claim formats are supported (`groups`, `roles`, provider-specific claims) + +### External Integration +External applications can query user group membership for implementing role-based access control: + +```bash +# View user groups via Headscale CLI +headscale users list --output json + +# Example database query (for direct database access) +SELECT name, email, groups FROM users WHERE provider = 'oidc'; +``` + +### Scope Requirements +To enable group storage, ensure your OIDC configuration includes the `groups` scope: + +```yaml +oidc: + issuer: "https://sso.example.com" + client_id: "headscale" + client_secret: "generated-secret" + scope: ["openid", "profile", "email", "groups"] +``` + +### Headplane Integration +When using [Headplane](https://github.com/tale/headplane) as a web interface for Headscale, OIDC groups enable automatic role-based access control: + +- **Automatic Role Assignment**: Users are assigned roles based on their OIDC group membership +- **Zero-Trust Security**: New users receive minimal access until proper groups are assigned +- **Dynamic Updates**: User roles update automatically on each login based on current group membership +- **Configurable Mapping**: Organizations can customize which groups map to which roles + +Example Headplane role mapping configuration: +```yaml +role_mapping: + owner: ["ceo", "cto", "headscale-owner"] + admin: ["it-admin", "platform-admin"] + network_admin: ["network-team", "devops"] + auditor: ["compliance", "audit-team"] +``` + +For detailed Headplane OIDC configuration, see the [Headplane documentation](https://github.com/tale/headplane/docs). ## Limitations - Support for OpenID Connect aims to be generic and vendor independent. It offers only limited support for quirks of specific identity providers. -- OIDC groups cannot be used in policy rules. +- OIDC groups cannot be used in policy rules directly (use external integrations for role-based access control). - The username provided by the identity provider needs to adhere to this pattern: - The username must be at least two characters long. - It must only contain letters, digits, hyphens, dots, underscores, and up to a single `@`. diff --git a/gen/go/headscale/v1/user.pb.go b/gen/go/headscale/v1/user.pb.go index 5f05d084..c1268e26 100644 --- a/gen/go/headscale/v1/user.pb.go +++ b/gen/go/headscale/v1/user.pb.go @@ -32,6 +32,10 @@ type User struct { ProviderId string `protobuf:"bytes,6,opt,name=provider_id,json=providerId,proto3" json:"provider_id,omitempty"` Provider string `protobuf:"bytes,7,opt,name=provider,proto3" json:"provider,omitempty"` ProfilePicUrl string `protobuf:"bytes,8,opt,name=profile_pic_url,json=profilePicUrl,proto3" json:"profile_pic_url,omitempty"` + // OIDC group memberships extracted from the identity provider's + // `groups` claim at login. Populated by hscontrol/types.User.FromClaim. + // External tools (Headplane, automation) use this for role-based access. + Groups []string `protobuf:"bytes,9,rep,name=groups,proto3" json:"groups,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -122,6 +126,13 @@ func (x *User) GetProfilePicUrl() string { return "" } +func (x *User) GetGroups() []string { + if x != nil { + return x.Groups + } + return nil +} + type CreateUserRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -518,7 +529,7 @@ var File_headscale_v1_user_proto protoreflect.FileDescriptor const file_headscale_v1_user_proto_rawDesc = "" + "\n" + - "\x17headscale/v1/user.proto\x12\fheadscale.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x83\x02\n" + + "\x17headscale/v1/user.proto\x12\fheadscale.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x9b\x02\n" + "\x04User\x12\x0e\n" + "\x02id\x18\x01 \x01(\x04R\x02id\x12\x12\n" + "\x04name\x18\x02 \x01(\tR\x04name\x129\n" + @@ -529,7 +540,8 @@ const file_headscale_v1_user_proto_rawDesc = "" + "\vprovider_id\x18\x06 \x01(\tR\n" + "providerId\x12\x1a\n" + "\bprovider\x18\a \x01(\tR\bprovider\x12&\n" + - "\x0fprofile_pic_url\x18\b \x01(\tR\rprofilePicUrl\"\x81\x01\n" + + "\x0fprofile_pic_url\x18\b \x01(\tR\rprofilePicUrl\x12\x16\n" + + "\x06groups\x18\t \x03(\tR\x06groups\"\x81\x01\n" + "\x11CreateUserRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12!\n" + "\fdisplay_name\x18\x02 \x01(\tR\vdisplayName\x12\x14\n" + diff --git a/gen/openapiv2/headscale/v1/headscale.swagger.json b/gen/openapiv2/headscale/v1/headscale.swagger.json index 545cf0b5..99480845 100644 --- a/gen/openapiv2/headscale/v1/headscale.swagger.json +++ b/gen/openapiv2/headscale/v1/headscale.swagger.json @@ -1528,6 +1528,13 @@ }, "profilePicUrl": { "type": "string" + }, + "groups": { + "type": "array", + "items": { + "type": "string" + }, + "description": "OIDC group memberships extracted from the identity provider's\n`groups` claim at login. Populated by hscontrol/types.User.FromClaim.\nExternal tools (Headplane, automation) use this for role-based access." } } } diff --git a/hscontrol/db/db.go b/hscontrol/db/db.go index 8c088c49..ad0dd6f6 100644 --- a/hscontrol/db/db.go +++ b/hscontrol/db/db.go @@ -215,6 +215,27 @@ AND auth_key_id NOT IN ( }, Rollback: func(db *gorm.DB) error { return nil }, }, + // Add groups column to users table for OIDC role mapping. + // Must run before any migration that loads users via the User struct + // (e.g., 202505141324), since the User struct now includes Groups. + { + ID: "202505141323", + Migrate: func(tx *gorm.DB) error { + if !tx.Migrator().HasColumn(&types.User{}, "groups") { + err := tx.Migrator().AddColumn(&types.User{}, "groups") + if err != nil { + return fmt.Errorf("adding groups column to users table: %w", err) + } + } + return nil + }, + Rollback: func(db *gorm.DB) error { + if db.Migrator().HasColumn(&types.User{}, "groups") { + return db.Migrator().DropColumn(&types.User{}, "groups") + } + return nil + }, + }, // Fix the provider identifier for users that have a double slash in the // provider identifier. { @@ -315,6 +336,7 @@ AND auth_key_id NOT IN ( provider_identifier text, provider text, profile_pic_url text, + groups text, created_at datetime, updated_at datetime, deleted_at datetime @@ -381,8 +403,8 @@ AND auth_key_id NOT IN ( // Copy data directly using SQL dataCopySQL := []string{ - `INSERT INTO users (id, name, display_name, email, provider_identifier, provider, profile_pic_url, created_at, updated_at, deleted_at) - SELECT id, name, display_name, email, provider_identifier, provider, profile_pic_url, created_at, updated_at, deleted_at + `INSERT INTO users (id, name, display_name, email, provider_identifier, provider, profile_pic_url, groups, created_at, updated_at, deleted_at) + SELECT id, name, display_name, email, provider_identifier, provider, profile_pic_url, groups, created_at, updated_at, deleted_at FROM users_old`, `INSERT INTO pre_auth_keys (id, key, user_id, reusable, ephemeral, used, tags, expiration, created_at) @@ -1000,6 +1022,7 @@ func runMigrations(cfg types.DatabaseConfig, dbConn *gorm.DB, migrations *gormig "202502131714", "202502171819", "202505091439", + "202505141323", "202505141324", // As of 2025-07-02, no new IDs should be added here. diff --git a/hscontrol/db/schema.sql b/hscontrol/db/schema.sql index 781446c0..d6511404 100644 --- a/hscontrol/db/schema.sql +++ b/hscontrol/db/schema.sql @@ -12,6 +12,7 @@ CREATE TABLE users( provider_identifier text, provider text, profile_pic_url text, + groups text, created_at datetime, updated_at datetime, diff --git a/hscontrol/types/types_clone.go b/hscontrol/types/types_clone.go index a14dc6fd..2eacb561 100644 --- a/hscontrol/types/types_clone.go +++ b/hscontrol/types/types_clone.go @@ -35,6 +35,7 @@ var _UserCloneNeedsRegeneration = User(struct { ProviderIdentifier sql.NullString Provider string ProfilePicURL string + Groups string }{}) // Clone makes a deep copy of Node. diff --git a/hscontrol/types/types_view.go b/hscontrol/types/types_view.go index 3a9b3b42..0db6b810 100644 --- a/hscontrol/types/types_view.go +++ b/hscontrol/types/types_view.go @@ -124,8 +124,11 @@ var _UserViewNeedsRegeneration = User(struct { ProviderIdentifier sql.NullString Provider string ProfilePicURL string + Groups string }{}) +func (v UserView) Groups() string { return v.ж.Groups } + // View returns a read-only view of Node. func (p *Node) View() NodeView { return NodeView{ж: p} diff --git a/hscontrol/types/users.go b/hscontrol/types/users.go index e5a9e7a5..b2455a67 100644 --- a/hscontrol/types/users.go +++ b/hscontrol/types/users.go @@ -95,6 +95,11 @@ type User struct { Provider string ProfilePicURL string + + // Groups stores the OIDC groups/roles that the user belongs to. + // This is populated from the 'groups' claim in OIDC tokens and + // is used for role-based access control in Headplane. + Groups string `gorm:"type:text"` } func (u *User) StringID() string { @@ -139,6 +144,39 @@ func (u *User) profilePicURL() string { return u.ProfilePicURL } +// GetGroups returns the user's groups as a slice of strings. +// Groups are stored as JSON in the database. +func (u *User) GetGroups() []string { + if u.Groups == "" { + return []string{} + } + + var groups []string + if err := json.Unmarshal([]byte(u.Groups), &groups); err != nil { + log.Error().Err(err).Msg("Failed to unmarshal user groups") + return []string{} + } + + return groups +} + +// SetGroups stores the user's groups as JSON in the database. +func (u *User) SetGroups(groups []string) { + if len(groups) == 0 { + u.Groups = "" + return + } + + data, err := json.Marshal(groups) + if err != nil { + log.Error().Err(err).Msg("Failed to marshal user groups") + u.Groups = "" + return + } + + u.Groups = string(data) +} + func (u *User) TailscaleUser() tailcfg.User { return tailcfg.User{ ID: tailcfg.UserID(u.ID), //nolint:gosec // UserID is bounded @@ -205,6 +243,7 @@ func (u *User) Proto() *v1.User { ProviderId: u.ProviderIdentifier.String, Provider: u.Provider, ProfilePicUrl: u.ProfilePicURL, + Groups: u.GetGroups(), } } @@ -447,4 +486,7 @@ func (u *User) FromClaim(claims *OIDCClaims, emailVerifiedRequired bool) { u.DisplayName = claims.Name u.ProfilePicURL = claims.ProfilePictureURL u.Provider = util.RegisterMethodOIDC + + // Store OIDC groups for role-based access control + u.SetGroups(claims.Groups) } diff --git a/hscontrol/types/users_test.go b/hscontrol/types/users_test.go index 1218979c..08823a3e 100644 --- a/hscontrol/types/users_test.go +++ b/hscontrol/types/users_test.go @@ -528,6 +528,7 @@ func TestOIDCClaimsJSONToUser(t *testing.T) { Valid: true, }, ProfilePicURL: "https://cdn.casbin.org/img/casbin.svg", + Groups: `["org1/department1","org1/department2"]`, }, }, } diff --git a/integration/oidc_groups_test.go b/integration/oidc_groups_test.go new file mode 100644 index 00000000..1695dac8 --- /dev/null +++ b/integration/oidc_groups_test.go @@ -0,0 +1,113 @@ +package integration + +import ( + "sort" + "testing" + + v1 "github.com/juanfont/headscale/gen/go/headscale/v1" + "github.com/juanfont/headscale/integration/hsic" + "github.com/oauth2-proxy/mockoidc" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOIDCGroupsPersisted verifies that the `groups` claim from an OIDC +// provider is persisted on the user and exposed through the gRPC API. +// +// Implementation under test: +// - users.groups TEXT column added by migration 202505141323. +// - hscontrol/types.User.SetGroups / GetGroups round-trip via JSON. +// - FromClaim calls SetGroups(claims.Groups) so login populates the +// column. OIDCClaims.Groups is FlexibleStringSlice, so providers like +// JumpCloud that return a single string instead of an array also work. +// - v1.User.Groups (proto field 9) is populated by User.Proto() so +// external tools (Headplane) can read group membership over gRPC. +func TestOIDCGroupsPersisted(t *testing.T) { + IntegrationSkip(t) + + // mockoidc serves logins from a strict queue, so keep NodesPerUser=1. + spec := ScenarioSpec{ + NodesPerUser: 1, + Users: []string{"admin", "dev", "solo"}, + OIDCUsers: []mockoidc.MockUser{ + oidcMockUserWithGroups("admin", true, []string{"admins", "engineering"}), + oidcMockUserWithGroups("dev", true, []string{"engineering"}), + // User with empty groups — round-trips as no Groups. + oidcMockUserWithGroups("solo", true, nil), + }, + } + + scenario, err := NewScenario(spec) + require.NoError(t, err) + defer scenario.ShutdownAssertNoPanics(t) + + oidcMap := map[string]string{ + "HEADSCALE_OIDC_ISSUER": scenario.mockOIDC.Issuer(), + "HEADSCALE_OIDC_CLIENT_ID": scenario.mockOIDC.ClientID(), + "CREDENTIALS_DIRECTORY_TEST": "/tmp", + "HEADSCALE_OIDC_CLIENT_SECRET_PATH": "${CREDENTIALS_DIRECTORY_TEST}/hs_client_oidc_secret", + // Make sure the OIDC scope set includes "groups" so mockoidc emits the claim. + "HEADSCALE_OIDC_SCOPE": "openid,profile,email,groups", + } + + err = scenario.CreateHeadscaleEnvWithLoginURL( + nil, + hsic.WithTestName("oidcgroups"), + hsic.WithConfigEnv(oidcMap), + hsic.WithFileInContainer("/tmp/hs_client_oidc_secret", []byte(scenario.mockOIDC.ClientSecret())), + ) + requireNoErrHeadscaleEnv(t, err) + + // Drive the OIDC login flow for every client. + _, err = scenario.ListTailscaleClients() + requireNoErrListClients(t, err) + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + headscale, err := scenario.Headscale() + require.NoError(t, err) + + users, err := headscale.ListUsers() + require.NoError(t, err) + + got := groupsByOIDCUserName(users) + + want := map[string][]string{ + "admin": {"admins", "engineering"}, + "dev": {"engineering"}, + "solo": nil, + } + + for name, wantGroups := range want { + gotGroups, ok := got[name] + assert.True(t, ok, "OIDC user %q not present in ListUsers response", name) + assert.ElementsMatch(t, wantGroups, gotGroups, "groups mismatch for user %q", name) + } +} + +// groupsByOIDCUserName picks out OIDC users from a ListUsers response and +// returns a map of username -> sorted groups. CLI-created users (no provider) +// are filtered out so the assertions don't need to know about them. +func groupsByOIDCUserName(users []*v1.User) map[string][]string { + out := map[string][]string{} + for _, u := range users { + if u.GetProvider() != "oidc" { + continue + } + g := append([]string(nil), u.GetGroups()...) + sort.Strings(g) + if len(g) == 0 { + g = nil + } + out[u.GetName()] = g + } + return out +} + +// oidcMockUserWithGroups extends [oidcMockUser] with a Groups claim. +// mockoidc populates the id_token / userinfo from this struct verbatim. +func oidcMockUserWithGroups(username string, emailVerified bool, groups []string) mockoidc.MockUser { + u := oidcMockUser(username, emailVerified) + u.Groups = groups + return u +} diff --git a/proto/headscale/v1/user.proto b/proto/headscale/v1/user.proto index bd71bcb1..a20587fe 100644 --- a/proto/headscale/v1/user.proto +++ b/proto/headscale/v1/user.proto @@ -13,6 +13,10 @@ message User { string provider_id = 6; string provider = 7; string profile_pic_url = 8; + // OIDC group memberships extracted from the identity provider's + // `groups` claim at login. Populated by hscontrol/types.User.FromClaim. + // External tools (Headplane, automation) use this for role-based access. + repeated string groups = 9; } message CreateUserRequest {