headscale/integration/hsic/hsic.go

1737 lines
47 KiB
Go
Raw Permalink Normal View History

package hsic
import (
2025-06-23 13:43:14 +02:00
"archive/tar"
"bytes"
2025-02-01 09:16:51 +00:00
"cmp"
2026-02-06 21:45:32 +01:00
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
2024-04-27 10:47:39 +02:00
"io"
"log"
"maps"
2026-02-06 21:45:32 +01:00
"net"
"net/http"
"net/netip"
"os"
"path"
2025-06-23 13:43:14 +02:00
"path/filepath"
2025-02-01 09:16:51 +00:00
"sort"
"strconv"
"strings"
"time"
"github.com/davecgh/go-spew/spew"
v1 "github.com/juanfont/headscale/gen/go/headscale/v1"
"github.com/juanfont/headscale/hscontrol"
policyv2 "github.com/juanfont/headscale/hscontrol/policy/v2"
"github.com/juanfont/headscale/hscontrol/types"
"github.com/juanfont/headscale/hscontrol/util"
"github.com/juanfont/headscale/integration/dockertestutil"
"github.com/juanfont/headscale/integration/integrationutil"
"github.com/ory/dockertest/v3"
"github.com/ory/dockertest/v3/docker"
"gopkg.in/yaml.v3"
"tailscale.com/tailcfg"
"tailscale.com/util/mak"
)
const (
hsicHashLength = 6
dockerContextPath = "../."
caCertRoot = "/usr/local/share/ca-certificates"
aclPolicyPath = "/etc/headscale/acl.hujson"
tlsCertPath = "/etc/headscale/tls.cert"
tlsKeyPath = "/etc/headscale/tls.key"
headscaleDefaultPort = 8080
IntegrationTestDockerFileName = "Dockerfile.integration"
2026-02-06 21:45:32 +01:00
defaultDirPerm = 0o755
binHeadscale = "headscale"
flagOutput = "--output"
acceptJSON = "Accept: application/json"
)
var (
errHeadscaleStatusCodeNotOk = errors.New("headscale status code not ok")
errInvalidHeadscaleImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_HEADSCALE_IMAGE format, expected repository:tag")
errHeadscaleImageRequiredInCI = errors.New("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE must be set in CI")
errInvalidPostgresImageFormat = errors.New("invalid HEADSCALE_INTEGRATION_POSTGRES_IMAGE format, expected repository:tag")
)
type fileInContainer struct {
path string
contents []byte
}
// HeadscaleInContainer is an implementation of ControlServer which
// sets up a Headscale instance inside a container.
type HeadscaleInContainer struct {
hostname string
pool *dockertest.Pool
container *dockertest.Resource
networks []*dockertest.Network
pgContainer *dockertest.Resource
// optional config
port int
extraPorts []string
hostMetricsPort string // Dynamically assigned host port for metrics/pprof access
caCerts [][]byte
hostPortBindings map[string][]string
aclPolicy *policyv2.Policy
env map[string]string
tlsCACert []byte
tlsCert []byte
tlsKey []byte
noTLS bool
filesInContainer []fileInContainer
postgres bool
policyMode types.PolicyMode
}
// Option represent optional settings that can be given to a
// Headscale instance.
type Option = func(c *HeadscaleInContainer)
// WithACLPolicy adds a [policyv2.Policy] to the
// [HeadscaleInContainer] instance.
func WithACLPolicy(acl *policyv2.Policy) Option {
return func(hsic *HeadscaleInContainer) {
if acl == nil {
return
}
// TODO(kradalby): Move somewhere appropriate
hsic.env["HEADSCALE_POLICY_PATH"] = aclPolicyPath
hsic.aclPolicy = acl
}
}
// WithCACert adds it to the trusted surtificate of the container.
func WithCACert(cert []byte) Option {
return func(hsic *HeadscaleInContainer) {
hsic.caCerts = append(hsic.caCerts, cert)
}
}
// WithoutTLS disables the default TLS configuration.
// Most tests should not need this. Use only for tests that
// explicitly need to test non-TLS behavior.
func WithoutTLS() Option {
return func(hsic *HeadscaleInContainer) {
hsic.noTLS = true
}
}
// WithCustomTLS uses the given certificates for the Headscale instance.
// The caCert is installed into the container's trust store and returned
// by [HeadscaleInContainer.GetCert] so that clients can trust this server.
func WithCustomTLS(caCert, cert, key []byte) Option {
return func(hsic *HeadscaleInContainer) {
hsic.tlsCACert = caCert
hsic.tlsCert = cert
hsic.tlsKey = key
hsic.caCerts = append(hsic.caCerts, caCert)
}
}
// WithConfigEnv takes a map of environment variables that
// can be used to override Headscale configuration.
func WithConfigEnv(configEnv map[string]string) Option {
return func(hsic *HeadscaleInContainer) {
maps.Copy(hsic.env, configEnv)
}
}
// WithPort sets the port on where to run Headscale.
func WithPort(port int) Option {
return func(hsic *HeadscaleInContainer) {
hsic.port = port
}
}
// WithExtraPorts exposes additional ports on the container (e.g. 3478/udp for STUN).
func WithExtraPorts(ports []string) Option {
return func(hsic *HeadscaleInContainer) {
hsic.extraPorts = ports
}
}
func WithHostPortBindings(bindings map[string][]string) Option {
return func(hsic *HeadscaleInContainer) {
hsic.hostPortBindings = bindings
}
}
// WithTestName sets a name for the test, this will be reflected
// in the Docker container name.
func WithTestName(testName string) Option {
return func(hsic *HeadscaleInContainer) {
hash, _ := util.GenerateRandomStringDNSSafe(hsicHashLength)
hostname := fmt.Sprintf("hs-%s-%s", testName, hash)
hsic.hostname = hostname
}
}
// WithHostname sets the hostname of the Headscale instance.
func WithHostname(hostname string) Option {
return func(hsic *HeadscaleInContainer) {
hsic.hostname = hostname
}
}
// WithFileInContainer adds a file to the container at the given path.
func WithFileInContainer(path string, contents []byte) Option {
return func(hsic *HeadscaleInContainer) {
hsic.filesInContainer = append(hsic.filesInContainer,
fileInContainer{
path: path,
contents: contents,
})
}
}
// WithPostgres spins up a Postgres container and
// sets it as the main database.
func WithPostgres() Option {
return func(hsic *HeadscaleInContainer) {
hsic.postgres = true
}
}
2026-02-06 21:45:32 +01:00
// WithPolicyMode sets the policy mode for headscale.
func WithPolicyMode(mode types.PolicyMode) Option {
return func(hsic *HeadscaleInContainer) {
hsic.policyMode = mode
hsic.env["HEADSCALE_POLICY_MODE"] = string(mode)
}
}
// WithIPAllocationStrategy sets the tests IP Allocation strategy.
func WithIPAllocationStrategy(strategy types.IPAllocationStrategy) Option {
return func(hsic *HeadscaleInContainer) {
hsic.env["HEADSCALE_PREFIXES_ALLOCATION"] = string(strategy)
}
}
// WithPublicDERP disables the embedded DERP server and restores
// the default public DERP relay configuration. Use this for tests
// that explicitly need to test public DERP behavior.
func WithPublicDERP() Option {
return func(hsic *HeadscaleInContainer) {
hsic.env["HEADSCALE_DERP_URLS"] = "https://controlplane.tailscale.com/derpmap/default"
hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false"
delete(hsic.env, "HEADSCALE_DERP_SERVER_REGION_ID")
delete(hsic.env, "HEADSCALE_DERP_SERVER_REGION_CODE")
delete(hsic.env, "HEADSCALE_DERP_SERVER_REGION_NAME")
delete(hsic.env, "HEADSCALE_DERP_SERVER_STUN_LISTEN_ADDR")
delete(hsic.env, "HEADSCALE_DERP_SERVER_PRIVATE_KEY_PATH")
delete(hsic.env, "DERP_DEBUG_LOGS")
delete(hsic.env, "DERP_PROBER_DEBUG_LOGS")
}
}
// WithDERPConfig configures Headscale use a custom
// DERP server only.
func WithDERPConfig(derpMap tailcfg.DERPMap) Option {
return func(hsic *HeadscaleInContainer) {
contents, err := yaml.Marshal(derpMap)
if err != nil {
log.Fatalf("marshalling DERP map: %s", err)
return
}
hsic.env["HEADSCALE_DERP_PATHS"] = "/etc/headscale/derp.yml"
hsic.filesInContainer = append(hsic.filesInContainer,
fileInContainer{
path: "/etc/headscale/derp.yml",
contents: contents,
})
// Disable global DERP server and embedded DERP server
hsic.env["HEADSCALE_DERP_URLS"] = ""
hsic.env["HEADSCALE_DERP_SERVER_ENABLED"] = "false"
// Envknob for enabling DERP debug logs
hsic.env["DERP_DEBUG_LOGS"] = "true"
hsic.env["DERP_PROBER_DEBUG_LOGS"] = "true"
}
}
// WithTuning allows changing the tuning settings easily.
func WithTuning(batchTimeout time.Duration, mapSessionChanSize int) Option {
return func(hsic *HeadscaleInContainer) {
hsic.env["HEADSCALE_TUNING_BATCH_CHANGE_DELAY"] = batchTimeout.String()
hsic.env["HEADSCALE_TUNING_NODE_MAPSESSION_BUFFERED_CHAN_SIZE"] = strconv.Itoa(
mapSessionChanSize,
)
}
}
func WithHAProbing(interval, timeout time.Duration) Option {
return func(hsic *HeadscaleInContainer) {
hsic.env["HEADSCALE_NODE_ROUTES_HA_PROBE_INTERVAL"] = interval.String()
hsic.env["HEADSCALE_NODE_ROUTES_HA_PROBE_TIMEOUT"] = timeout.String()
}
}
func WithTimezone(timezone string) Option {
return func(hsic *HeadscaleInContainer) {
hsic.env["TZ"] = timezone
}
}
// buildEntrypoint builds the container entrypoint command based on configuration.
// It constructs proper wait conditions instead of fixed sleeps:
// 1. Wait for network to be ready
// 2. Wait for config.yaml (always written after container start)
// 3. Wait for CA certs if configured
// 4. Update CA certificates
// 5. Run headscale serve
// 6. Sleep at end to keep container alive for log collection on shutdown.
func (hsic *HeadscaleInContainer) buildEntrypoint() []string {
var commands []string
// Wait for network to be ready
commands = append(commands, "while ! ip route show default >/dev/null 2>&1; do sleep 0.1; done")
// Wait for config.yaml to be written (always written after container start)
commands = append(commands, "while [ ! -f /etc/headscale/config.yaml ]; do sleep 0.1; done")
// If CA certs are configured, wait for them to be written
if len(hsic.caCerts) > 0 {
commands = append(commands,
fmt.Sprintf("while [ ! -f %s/user-0.crt ]; do sleep 0.1; done", caCertRoot))
}
// Update CA certificates
commands = append(commands, "update-ca-certificates")
// Run headscale serve
commands = append(commands, "/usr/local/bin/headscale serve")
// Keep container alive after headscale exits for log collection
commands = append(commands, "/bin/sleep 30")
return []string{"/bin/bash", "-c", strings.Join(commands, " ; ")}
}
// New returns a new [HeadscaleInContainer] instance.
2026-02-06 21:45:32 +01:00
//
//nolint:gocyclo // complex container setup with many options
func New(
pool *dockertest.Pool,
networks []*dockertest.Network,
opts ...Option,
) (*HeadscaleInContainer, error) {
hash, err := util.GenerateRandomStringDNSSafe(hsicHashLength)
if err != nil {
return nil, err
}
// Include run ID in hostname for easier identification of which test run owns this container
runID := dockertestutil.GetIntegrationRunID()
var hostname string
if runID != "" {
// Use last 6 chars of run ID (the random hash part) for brevity
runIDShort := runID[len(runID)-6:]
hostname = fmt.Sprintf("hs-%s-%s", runIDShort, hash)
} else {
hostname = "hs-" + hash
}
hsic := &HeadscaleInContainer{
hostname: hostname,
port: headscaleDefaultPort,
pool: pool,
networks: networks,
env: DefaultConfigEnv(),
filesInContainer: []fileInContainer{},
policyMode: types.PolicyModeFile,
}
for _, opt := range opts {
opt(hsic)
}
// TLS is enabled by default for all integration tests.
// Generate a self-signed certificate if TLS was not explicitly
// disabled via [WithoutTLS] and no custom cert was provided
// via [WithCustomTLS].
if !hsic.noTLS && len(hsic.tlsCert) == 0 {
caCert, cert, key, err := integrationutil.CreateCertificate(hsic.hostname)
if err != nil {
return nil, fmt.Errorf("creating default TLS certificates: %w", err)
}
hsic.tlsCACert = caCert
hsic.tlsCert = cert
hsic.tlsKey = key
// Install the CA cert into the headscale container's trust
// store so that tools like curl trust the server's own
// certificate.
hsic.caCerts = append(hsic.caCerts, caCert)
}
log.Println("NAME: ", hsic.hostname)
portProto := fmt.Sprintf("%d/tcp", hsic.port)
headscaleBuildOptions := &dockertest.BuildOptions{
Dockerfile: IntegrationTestDockerFileName,
ContextDir: dockerContextPath,
}
if hsic.postgres {
hsic.env["HEADSCALE_DATABASE_TYPE"] = "postgres"
hsic.env["HEADSCALE_DATABASE_POSTGRES_HOST"] = "postgres-" + hash
hsic.env["HEADSCALE_DATABASE_POSTGRES_USER"] = binHeadscale
hsic.env["HEADSCALE_DATABASE_POSTGRES_PASS"] = binHeadscale
hsic.env["HEADSCALE_DATABASE_POSTGRES_NAME"] = binHeadscale
delete(hsic.env, "HEADSCALE_DATABASE_SQLITE_PATH")
// Determine postgres image - use prebuilt if available, otherwise pull from registry
pgRepo := "postgres"
pgTag := "latest"
if prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_POSTGRES_IMAGE"); prebuiltImage != "" {
repo, tag, found := strings.Cut(prebuiltImage, ":")
if !found {
return nil, errInvalidPostgresImageFormat
}
pgRepo = repo
pgTag = tag
}
2025-06-23 13:43:14 +02:00
pgRunOptions := &dockertest.RunOptions{
Name: "postgres-" + hash,
Repository: pgRepo,
Tag: pgTag,
2025-06-23 13:43:14 +02:00
Networks: networks,
Env: []string{
"POSTGRES_USER=headscale",
"POSTGRES_PASSWORD=headscale",
"POSTGRES_DB=headscale",
},
}
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(pgRunOptions, "postgres")
2025-06-23 13:43:14 +02:00
pg, err := pool.RunWithOptions(pgRunOptions)
if err != nil {
return nil, fmt.Errorf("starting postgres container: %w", err)
}
hsic.pgContainer = pg
}
env := []string{
"HEADSCALE_DEBUG_PROFILING_ENABLED=1",
"HEADSCALE_DEBUG_PROFILING_PATH=/tmp/profile",
"HEADSCALE_DEBUG_DUMP_MAPRESPONSE_PATH=/tmp/mapresponses",
"HEADSCALE_DEBUG_DEADLOCK=1",
"HEADSCALE_DEBUG_DEADLOCK_TIMEOUT=5s",
"HEADSCALE_DEBUG_HIGH_CARDINALITY_METRICS=1",
"HEADSCALE_DEBUG_DUMP_CONFIG=1",
}
if hsic.hasTLS() {
hsic.env["HEADSCALE_TLS_CERT_PATH"] = tlsCertPath
hsic.env["HEADSCALE_TLS_KEY_PATH"] = tlsKeyPath
}
// Server URL and Listen Addr should not be overridable outside of
// the configuration passed to docker.
hsic.env["HEADSCALE_SERVER_URL"] = hsic.GetEndpoint()
hsic.env["HEADSCALE_LISTEN_ADDR"] = fmt.Sprintf("0.0.0.0:%d", hsic.port)
for key, value := range hsic.env {
env = append(env, fmt.Sprintf("%s=%s", key, value))
}
log.Printf("ENV: \n%s", spew.Sdump(hsic.env))
runOptions := &dockertest.RunOptions{
Name: hsic.hostname,
ExposedPorts: append([]string{portProto, "9090/tcp"}, hsic.extraPorts...),
Networks: networks,
// Cmd: []string{"headscale", "serve"},
// TODO(kradalby): Get rid of this hack, we currently need to give us some
// to inject the headscale configuration further down.
Entrypoint: hsic.buildEntrypoint(),
Env: env,
}
// Bind metrics port to dynamic host port (kernel assigns free port)
if runOptions.PortBindings == nil {
runOptions.PortBindings = map[docker.Port][]docker.PortBinding{}
}
runOptions.PortBindings["9090/tcp"] = []docker.PortBinding{
{HostPort: "0"}, // Let kernel assign a free port
}
if len(hsic.hostPortBindings) > 0 {
for port, hostPorts := range hsic.hostPortBindings {
runOptions.PortBindings[docker.Port(port)] = []docker.PortBinding{}
for _, hostPort := range hostPorts {
runOptions.PortBindings[docker.Port(port)] = append(
runOptions.PortBindings[docker.Port(port)],
docker.PortBinding{HostPort: hostPort})
}
}
}
2025-02-05 16:10:18 +01:00
// dockertest isn't very good at handling containers that has already
// been created, this is an attempt to make sure this container isn't
// present.
err = pool.RemoveContainerByName(hsic.hostname)
if err != nil {
return nil, err
}
2025-06-23 13:43:14 +02:00
// Add integration test labels if running under hi tool
dockertestutil.DockerAddIntegrationLabels(runOptions, binHeadscale)
var container *dockertest.Resource
// Check if a pre-built image is available via environment variable
prebuiltImage := os.Getenv("HEADSCALE_INTEGRATION_HEADSCALE_IMAGE")
if prebuiltImage != "" {
log.Printf("Using pre-built headscale image: %s", prebuiltImage) //nolint:gosec // G706: integration-only log of trusted env value
// Parse image into repository and tag
repo, tag, ok := strings.Cut(prebuiltImage, ":")
if !ok {
return nil, errInvalidHeadscaleImageFormat
}
runOptions.Repository = repo
runOptions.Tag = tag
container, err = pool.RunWithOptions(
runOptions,
dockertestutil.DockerRestartPolicy,
dockertestutil.DockerAllowLocalIPv6,
dockertestutil.DockerAllowNetworkAdministration,
)
if err != nil {
return nil, fmt.Errorf("running pre-built headscale container %q: %w", prebuiltImage, err)
}
} else if util.IsCI() {
return nil, errHeadscaleImageRequiredInCI
} else {
container, err = pool.BuildAndRunWithBuildOptions(
headscaleBuildOptions,
runOptions,
dockertestutil.DockerRestartPolicy,
dockertestutil.DockerAllowLocalIPv6,
dockertestutil.DockerAllowNetworkAdministration,
)
if err != nil {
// Try to get more detailed build output
log.Printf("Docker build/run failed, attempting to get detailed output...")
buildOutput, buildErr := dockertestutil.RunDockerBuildForDiagnostics(dockerContextPath, IntegrationTestDockerFileName)
// Show the last 100 lines of build output to avoid overwhelming the logs
lines := strings.Split(buildOutput, "\n")
const maxLines = 100
startLine := 0
if len(lines) > maxLines {
startLine = len(lines) - maxLines
}
relevantOutput := strings.Join(lines[startLine:], "\n")
if buildErr != nil {
// The diagnostic build also failed - this is the real error
return nil, fmt.Errorf("starting headscale container: %w\n\nDocker build failed. Last %d lines of output:\n%s", err, maxLines, relevantOutput)
}
if buildOutput != "" {
// Build succeeded on retry but container creation still failed
return nil, fmt.Errorf("starting headscale container: %w\n\nDocker build succeeded on retry, but container creation failed. Last %d lines of build output:\n%s", err, maxLines, relevantOutput)
}
// No output at all - diagnostic build command may have failed
return nil, fmt.Errorf("starting headscale container: %w\n\nUnable to get diagnostic build output (command may have failed silently)", err)
}
}
2026-02-06 21:45:32 +01:00
log.Printf("Created %s container\n", hsic.hostname)
hsic.container = container
// Get the dynamically assigned host port for metrics/pprof
hsic.hostMetricsPort = container.GetHostPort("9090/tcp")
log.Printf(
"Headscale %s metrics available at http://localhost:%s/metrics (debug at http://localhost:%s/debug/)\n",
hsic.hostname,
hsic.hostMetricsPort,
hsic.hostMetricsPort,
)
// Write the CA certificates to the container
for i, cert := range hsic.caCerts {
err = hsic.WriteFile(fmt.Sprintf("%s/user-%d.crt", caCertRoot, i), cert)
if err != nil {
return nil, fmt.Errorf("writing TLS certificate to container: %w", err)
}
}
err = hsic.WriteFile("/etc/headscale/config.yaml", []byte(MinimumConfigYAML()))
if err != nil {
return nil, fmt.Errorf("writing headscale config to container: %w", err)
}
if hsic.aclPolicy != nil {
err = hsic.writePolicy(hsic.aclPolicy)
if err != nil {
return nil, fmt.Errorf("writing policy: %w", err)
}
}
if hsic.hasTLS() {
err = hsic.WriteFile(tlsCertPath, hsic.tlsCert)
if err != nil {
return nil, fmt.Errorf("writing TLS certificate to container: %w", err)
}
err = hsic.WriteFile(tlsKeyPath, hsic.tlsKey)
if err != nil {
return nil, fmt.Errorf("writing TLS key to container: %w", err)
}
}
for _, f := range hsic.filesInContainer {
2026-02-06 21:45:32 +01:00
err := hsic.WriteFile(f.path, f.contents)
if err != nil {
return nil, fmt.Errorf("writing %q: %w", f.path, err)
}
}
// Load the database from policy file on repeat until it succeeds,
// this is done as the container sleeps before starting headscale.
if hsic.aclPolicy != nil && hsic.policyMode == types.PolicyModeDB {
err := pool.Retry(hsic.reloadDatabasePolicy)
if err != nil {
return nil, fmt.Errorf("loading database policy on startup: %w", err)
}
}
return hsic, nil
}
func (t *HeadscaleInContainer) ConnectToNetwork(network *dockertest.Network) error {
return t.container.ConnectToNetwork(network)
}
func (t *HeadscaleInContainer) hasTLS() bool {
return len(t.tlsCert) != 0 && len(t.tlsKey) != 0
}
// Shutdown stops and cleans up the Headscale container.
func (t *HeadscaleInContainer) Shutdown() (string, string, error) {
stdoutPath, stderrPath, err := t.SaveLog("/tmp/control")
if err != nil {
log.Printf(
"saving log from control: %s",
fmt.Errorf("saving log from control: %w", err),
)
}
err = t.SaveMetrics(fmt.Sprintf("/tmp/control/%s_metrics.txt", t.hostname))
2024-04-27 10:47:39 +02:00
if err != nil {
log.Printf(
"saving metrics from control: %s",
2024-04-27 10:47:39 +02:00
err,
)
}
// Send a interrupt signal to the "headscale" process inside the container
// allowing it to shut down gracefully and flush the profile to disk.
// The container will live for a bit longer due to the sleep at the end.
err = t.SendInterrupt()
if err != nil {
log.Printf(
"sending graceful interrupt to control: %s",
fmt.Errorf("sending graceful interrupt to control: %w", err),
)
}
err = t.SaveProfile("/tmp/control")
if err != nil {
log.Printf(
"saving profile from control: %s",
fmt.Errorf("saving profile from control: %w", err),
)
}
err = t.SaveMapResponses("/tmp/control")
if err != nil {
log.Printf(
"saving mapresponses from control: %s",
fmt.Errorf("saving mapresponses from control: %w", err),
)
}
// We dont have a database to save if we use postgres
if !t.postgres {
err = t.SaveDatabase("/tmp/control")
if err != nil {
log.Printf(
"saving database from control: %s",
fmt.Errorf("saving database from control: %w", err),
)
}
}
// Cleanup postgres container if enabled.
if t.postgres {
2026-02-06 21:45:32 +01:00
_ = t.pool.Purge(t.pgContainer)
}
return stdoutPath, stderrPath, t.pool.Purge(t.container)
}
// WriteLogs writes the current stdout/stderr log of the container to
// the given [io.Writer]s.
func (t *HeadscaleInContainer) WriteLogs(stdout, stderr io.Writer) error {
return dockertestutil.WriteLog(t.pool, t.container, stdout, stderr)
}
// ReadLog returns the current stdout and stderr logs from the headscale container.
func (t *HeadscaleInContainer) ReadLog() (string, string, error) {
var stdout, stderr bytes.Buffer
err := dockertestutil.WriteLog(t.pool, t.container, &stdout, &stderr)
if err != nil {
return "", "", fmt.Errorf("reading container logs: %w", err)
}
return stdout.String(), stderr.String(), nil
}
// SaveLog saves the current stdout log of the container to a path
// on the host system.
func (t *HeadscaleInContainer) SaveLog(path string) (string, string, error) {
return dockertestutil.SaveLog(t.pool, t.container, path)
}
2024-04-27 10:47:39 +02:00
func (t *HeadscaleInContainer) SaveMetrics(savePath string) error {
2026-02-06 21:45:32 +01:00
req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, "http://"+net.JoinHostPort(t.hostname, "9090")+"/metrics", nil)
if err != nil {
return fmt.Errorf("creating metrics request: %w", err)
}
resp, err := http.DefaultClient.Do(req)
2024-04-27 10:47:39 +02:00
if err != nil {
return fmt.Errorf("getting metrics: %w", err)
}
defer resp.Body.Close()
2026-02-06 21:45:32 +01:00
2024-04-27 10:47:39 +02:00
out, err := os.Create(savePath)
if err != nil {
return fmt.Errorf("creating file for metrics: %w", err)
}
defer out.Close()
2026-02-06 21:45:32 +01:00
2024-04-27 10:47:39 +02:00
_, err = io.Copy(out, resp.Body)
if err != nil {
return fmt.Errorf("copy response to file: %w", err)
}
return nil
}
2025-06-23 13:43:14 +02:00
// extractTarToDirectory extracts a tar archive to a directory.
func extractTarToDirectory(tarData []byte, targetDir string) error {
2026-02-06 21:45:32 +01:00
err := os.MkdirAll(targetDir, defaultDirPerm)
if err != nil {
return fmt.Errorf("creating directory %s: %w", targetDir, err)
2025-06-23 13:43:14 +02:00
}
2025-08-27 17:09:13 +02:00
// Find the top-level directory to strip
var topLevelDir string
2026-02-06 21:45:32 +01:00
2025-08-27 17:09:13 +02:00
firstPass := tar.NewReader(bytes.NewReader(tarData))
for {
header, err := firstPass.Next()
if err == io.EOF {
break
}
2026-02-06 21:45:32 +01:00
2025-08-27 17:09:13 +02:00
if err != nil {
return fmt.Errorf("reading tar header: %w", err)
2025-08-27 17:09:13 +02:00
}
if header.Typeflag == tar.TypeDir && topLevelDir == "" {
topLevelDir = strings.TrimSuffix(header.Name, "/")
break
}
}
2026-02-06 21:45:32 +01:00
tarReader := tar.NewReader(bytes.NewReader(tarData))
2025-06-23 13:43:14 +02:00
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
if err != nil {
return fmt.Errorf("reading tar header: %w", err)
2025-06-23 13:43:14 +02:00
}
// Clean the path to prevent directory traversal
cleanName := filepath.Clean(header.Name)
if strings.Contains(cleanName, "..") {
continue // Skip potentially dangerous paths
}
2025-08-27 17:09:13 +02:00
// Strip the top-level directory
if topLevelDir != "" && strings.HasPrefix(cleanName, topLevelDir+"/") {
cleanName = strings.TrimPrefix(cleanName, topLevelDir+"/")
} else if cleanName == topLevelDir {
// Skip the top-level directory itself
continue
}
2025-08-27 17:09:13 +02:00
// Skip empty paths after stripping
if cleanName == "" {
continue
}
targetPath := filepath.Join(targetDir, cleanName)
2025-06-23 13:43:14 +02:00
switch header.Typeflag {
case tar.TypeDir:
// Create directory
2026-02-06 21:45:32 +01:00
//nolint:gosec // G115: header.Mode is trusted from tar archive
err := os.MkdirAll(targetPath, os.FileMode(header.Mode))
if err != nil {
return fmt.Errorf("creating directory %s: %w", targetPath, err)
2025-06-23 13:43:14 +02:00
}
case tar.TypeReg:
2025-08-27 17:09:13 +02:00
// Ensure parent directories exist
2026-02-06 21:45:32 +01:00
err := os.MkdirAll(filepath.Dir(targetPath), defaultDirPerm)
if err != nil {
return fmt.Errorf("creating parent directories for %s: %w", targetPath, err)
2025-08-27 17:09:13 +02:00
}
2025-06-23 13:43:14 +02:00
// Create file
outFile, err := os.Create(targetPath)
if err != nil {
return fmt.Errorf("creating file %s: %w", targetPath, err)
2025-06-23 13:43:14 +02:00
}
2026-02-06 21:45:32 +01:00
if _, err := io.Copy(outFile, tarReader); err != nil { //nolint:gosec,noinlineerr // trusted tar from test container
2025-06-23 13:43:14 +02:00
outFile.Close()
return fmt.Errorf("copying file contents: %w", err)
2025-06-23 13:43:14 +02:00
}
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
outFile.Close()
// Set file permissions
2026-02-06 21:45:32 +01:00
if err := os.Chmod(targetPath, os.FileMode(header.Mode)); err != nil { //nolint:gosec,noinlineerr // safe mode from tar header
return fmt.Errorf("setting file permissions: %w", err)
2025-06-23 13:43:14 +02:00
}
}
}
return nil
}
func (t *HeadscaleInContainer) SaveProfile(savePath string) error {
tarFile, err := t.FetchPath("/tmp/profile")
if err != nil {
return err
}
2025-08-27 17:09:13 +02:00
targetDir := path.Join(savePath, "pprof")
2025-06-23 13:43:14 +02:00
return extractTarToDirectory(tarFile, targetDir)
}
func (t *HeadscaleInContainer) SaveMapResponses(savePath string) error {
tarFile, err := t.FetchPath("/tmp/mapresponses")
if err != nil {
return err
}
2025-08-27 17:09:13 +02:00
targetDir := path.Join(savePath, "mapresponses")
2025-06-23 13:43:14 +02:00
return extractTarToDirectory(tarFile, targetDir)
}
2025-06-23 13:43:14 +02:00
func (t *HeadscaleInContainer) SaveDatabase(savePath string) error {
// If using PostgreSQL, skip database file extraction
if t.postgres {
return nil
}
// Also check for any .sqlite files
sqliteFiles, err := t.Execute([]string{"find", "/tmp", "-name", "*.sqlite*", "-type", "f"})
if err != nil {
2025-06-23 13:43:14 +02:00
log.Printf("Warning: could not find sqlite files: %v", err)
} else {
log.Printf("SQLite files found in %s:\n%s", t.hostname, sqliteFiles)
}
2025-06-23 13:43:14 +02:00
// Check if the database file exists and has a schema
dbPath := "/tmp/integration_test_db.sqlite3"
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
fileInfo, err := t.Execute([]string{"ls", "-la", dbPath})
if err != nil {
return fmt.Errorf("database file does not exist at %s: %w", dbPath, err)
}
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
log.Printf("Database file info: %s", fileInfo)
2025-06-23 13:43:14 +02:00
// Check if the database has any tables (schema)
schemaCheck, err := t.Execute([]string{"sqlite3", dbPath, ".schema"})
if err != nil {
return fmt.Errorf("checking database schema (sqlite3 command failed): %w", err)
}
2025-06-23 13:43:14 +02:00
if strings.TrimSpace(schemaCheck) == "" {
2026-02-06 21:45:32 +01:00
return errors.New("database file exists but has no schema (empty database)") //nolint:err113
2025-06-23 13:43:14 +02:00
}
2025-06-23 13:43:14 +02:00
tarFile, err := t.FetchPath("/tmp/integration_test_db.sqlite3")
if err != nil {
return fmt.Errorf("fetching database file: %w", err)
}
2025-06-23 13:43:14 +02:00
// For database, extract the first regular file (should be the SQLite file)
tarReader := tar.NewReader(bytes.NewReader(tarFile))
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
if err != nil {
return fmt.Errorf("reading tar header: %w", err)
2025-06-23 13:43:14 +02:00
}
log.Printf(
"Found file in tar: %s (type: %d, size: %d)",
header.Name,
header.Typeflag,
header.Size,
)
2025-06-23 13:43:14 +02:00
// Extract the first regular file we find
if header.Typeflag == tar.TypeReg {
dbPath := path.Join(savePath, t.hostname+".db")
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
outFile, err := os.Create(dbPath)
if err != nil {
return fmt.Errorf("creating database file: %w", err)
2025-06-23 13:43:14 +02:00
}
2026-02-06 21:45:32 +01:00
written, err := io.Copy(outFile, tarReader) //nolint:gosec // trusted tar from test container
2025-06-23 13:43:14 +02:00
outFile.Close()
2026-02-06 21:45:32 +01:00
2025-06-23 13:43:14 +02:00
if err != nil {
return fmt.Errorf("copying database file: %w", err)
2025-06-23 13:43:14 +02:00
}
log.Printf(
"Extracted database file: %s (%d bytes written, header claimed %d bytes)",
dbPath,
written,
header.Size,
)
2025-06-23 13:43:14 +02:00
// Check if we actually wrote something
if written == 0 {
2026-02-06 21:45:32 +01:00
return fmt.Errorf( //nolint:err113
"database file is empty (size: %d, header size: %d)",
written,
header.Size,
)
2025-06-23 13:43:14 +02:00
}
return nil
}
}
2026-02-06 21:45:32 +01:00
return errors.New("no regular file found in database tar archive") //nolint:err113
}
// Execute runs a command inside the Headscale container and returns the
// result of stdout as a string.
func (t *HeadscaleInContainer) Execute(
command []string,
) (string, error) {
stdout, stderr, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
log.Printf("command: %v", command)
log.Printf("command stderr: %s\n", stderr)
if stdout != "" {
log.Printf("command stdout: %s\n", stdout)
}
return stdout, fmt.Errorf("executing command in docker: %w, stderr: %s", err, stderr)
}
return stdout, nil
}
// GetPort returns the docker container port as a string.
func (t *HeadscaleInContainer) GetPort() string {
return strconv.Itoa(t.port)
}
// GetHostMetricsPort returns the dynamically assigned host port for metrics/pprof access.
// This port can be used by operators to access metrics at http://localhost:{port}/metrics
// and debug endpoints at http://localhost:{port}/debug/ while tests are running.
func (t *HeadscaleInContainer) GetHostMetricsPort() string {
return t.hostMetricsPort
}
// GetHealthEndpoint returns a health endpoint for the [HeadscaleInContainer]
// instance.
func (t *HeadscaleInContainer) GetHealthEndpoint() string {
return t.GetEndpoint() + "/health"
}
// GetEndpoint returns the Headscale endpoint for the [HeadscaleInContainer].
func (t *HeadscaleInContainer) GetEndpoint() string {
return t.getEndpoint(false)
}
// GetIPEndpoint returns the Headscale endpoint using IP address instead of hostname.
func (t *HeadscaleInContainer) GetIPEndpoint() string {
return t.getEndpoint(true)
}
// getEndpoint returns the Headscale endpoint, optionally using IP address instead of hostname.
func (t *HeadscaleInContainer) getEndpoint(useIP bool) string {
var host string
if useIP && len(t.networks) > 0 {
// Use IP address from the first network
host = t.GetIPInNetwork(t.networks[0])
} else {
host = t.GetHostname()
}
hostEndpoint := fmt.Sprintf("%s:%d", host, t.port)
if t.hasTLS() {
return "https://" + hostEndpoint
}
return "http://" + hostEndpoint
}
// GetCert returns the CA certificate that clients should trust to
// verify this server's TLS certificate.
func (t *HeadscaleInContainer) GetCert() []byte {
return t.tlsCACert
}
// GetHostname returns the hostname of the [HeadscaleInContainer].
func (t *HeadscaleInContainer) GetHostname() string {
return t.hostname
}
// GetIPInNetwork returns the IP address of the [HeadscaleInContainer] in the given network.
func (t *HeadscaleInContainer) GetIPInNetwork(network *dockertest.Network) string {
return t.container.GetIPInNetwork(network)
}
// WaitForRunning blocks until the Headscale instance is ready to
// serve clients.
func (t *HeadscaleInContainer) WaitForRunning() error {
url := t.GetHealthEndpoint()
log.Printf("waiting for headscale to be ready at %s", url)
client := &http.Client{}
if t.hasTLS() {
insecureTransport := http.DefaultTransport.(*http.Transport).Clone() //nolint
insecureTransport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint
client = &http.Client{Transport: insecureTransport}
}
return t.pool.Retry(func() error {
resp, err := client.Get(url) //nolint
if err != nil {
return fmt.Errorf("headscale is not ready: %w", err)
}
if resp.StatusCode != http.StatusOK {
return errHeadscaleStatusCodeNotOk
}
return nil
})
}
// CreateUser adds a new user to the Headscale instance.
func (t *HeadscaleInContainer) CreateUser(
user string,
) (*v1.User, error) {
command := []string{
binHeadscale,
"users",
"create",
user,
fmt.Sprintf("--email=%s@test.no", user),
flagOutput,
"json",
}
result, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return nil, err
}
var u v1.User
2026-02-06 21:45:32 +01:00
err = json.Unmarshal([]byte(result), &u)
if err != nil {
return nil, fmt.Errorf("unmarshalling user: %w", err)
}
return &u, nil
}
// AuthKeyOptions defines options for creating an auth key.
type AuthKeyOptions struct {
// User is the user ID that owns the auth key. If nil and Tags are specified,
// the auth key is owned by the tags only (tags-as-identity model).
User *uint64
// Reusable indicates if the key can be used multiple times
Reusable bool
// Ephemeral indicates if nodes registered with this key should be ephemeral
Ephemeral bool
// Tags are the tags to assign to the auth key
Tags []string
}
// CreateAuthKeyWithOptions creates a new "authorisation key" with the specified options.
// This supports both user-owned and tags-only auth keys.
func (t *HeadscaleInContainer) CreateAuthKeyWithOptions(opts AuthKeyOptions) (*v1.PreAuthKey, error) {
command := []string{
binHeadscale,
}
// Only add --user flag if User is specified
if opts.User != nil {
command = append(command, "--user", strconv.FormatUint(*opts.User, 10))
}
command = append(command,
"preauthkeys",
"create",
"--expiration",
"24h",
flagOutput,
"json",
)
if opts.Reusable {
command = append(command, "--reusable")
}
if opts.Ephemeral {
command = append(command, "--ephemeral")
}
if len(opts.Tags) > 0 {
command = append(command, "--tags", strings.Join(opts.Tags, ","))
}
result, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return nil, fmt.Errorf("executing create auth key command: %w", err)
}
var preAuthKey v1.PreAuthKey
err = json.Unmarshal([]byte(result), &preAuthKey)
if err != nil {
return nil, fmt.Errorf("unmarshalling auth key: %w", err)
}
return &preAuthKey, nil
}
// CreateAuthKey creates a new "authorisation key" for a User that can be used
// to authorise a TailscaleClient with the [HeadscaleInContainer] instance.
func (t *HeadscaleInContainer) CreateAuthKey(
user uint64,
reusable bool,
ephemeral bool,
) (*v1.PreAuthKey, error) {
return t.CreateAuthKeyWithOptions(AuthKeyOptions{
User: &user,
Reusable: reusable,
Ephemeral: ephemeral,
})
}
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
// CreateAuthKeyWithTags creates a new "authorisation key" for a User with the specified tags.
// This is used to create tagged PreAuthKeys for testing the tags-as-identity model.
func (t *HeadscaleInContainer) CreateAuthKeyWithTags(
user uint64,
reusable bool,
ephemeral bool,
tags []string,
) (*v1.PreAuthKey, error) {
return t.CreateAuthKeyWithOptions(AuthKeyOptions{
User: &user,
Reusable: reusable,
Ephemeral: ephemeral,
Tags: tags,
})
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
}
// DeleteAuthKey deletes an "authorisation key" by ID.
func (t *HeadscaleInContainer) DeleteAuthKey(
id uint64,
) error {
command := []string{
binHeadscale,
"preauthkeys",
"delete",
"--id",
strconv.FormatUint(id, 10),
flagOutput,
"json",
}
_, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return fmt.Errorf("executing delete auth key command: %w", err)
}
return nil
}
2025-02-01 09:16:51 +00:00
// ListNodes lists the currently registered Nodes in headscale.
// Optionally a list of usernames can be passed to get users for
// specific users.
func (t *HeadscaleInContainer) ListNodes(
users ...string,
2023-09-24 13:42:05 +02:00
) ([]*v1.Node, error) {
2025-02-01 09:16:51 +00:00
var ret []*v1.Node
2026-02-06 21:45:32 +01:00
2025-02-01 09:16:51 +00:00
execUnmarshal := func(command []string) error {
result, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return fmt.Errorf("executing list node command: %w", err)
2025-02-01 09:16:51 +00:00
}
var nodes []*v1.Node
2026-02-06 21:45:32 +01:00
2025-02-01 09:16:51 +00:00
err = json.Unmarshal([]byte(result), &nodes)
if err != nil {
return fmt.Errorf("unmarshalling nodes: %w", err)
2025-02-01 09:16:51 +00:00
}
ret = append(ret, nodes...)
2025-02-01 09:16:51 +00:00
return nil
}
if len(users) == 0 {
err := execUnmarshal([]string{binHeadscale, "nodes", "list", flagOutput, "json"})
2025-02-01 09:16:51 +00:00
if err != nil {
return nil, err
}
} else {
for _, user := range users {
command := []string{binHeadscale, "--user", user, "nodes", "list", flagOutput, "json"}
2025-02-01 09:16:51 +00:00
err := execUnmarshal(command)
if err != nil {
return nil, err
}
}
}
sort.Slice(ret, func(i, j int) bool {
return cmp.Compare(ret[i].GetId(), ret[j].GetId()) == -1
})
2025-02-01 09:16:51 +00:00
return ret, nil
}
func (t *HeadscaleInContainer) DeleteNode(nodeID uint64) error {
command := []string{
binHeadscale,
"nodes",
"delete",
"--identifier",
2026-02-06 21:45:32 +01:00
strconv.FormatUint(nodeID, 10),
flagOutput,
"json",
"--force",
}
_, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return fmt.Errorf("executing delete node command: %w", err)
}
return nil
}
func (t *HeadscaleInContainer) NodesByUser() (map[string][]*v1.Node, error) {
nodes, err := t.ListNodes()
if err != nil {
return nil, err
}
var userMap map[string][]*v1.Node
for _, node := range nodes {
if _, ok := userMap[node.GetUser().GetName()]; !ok {
mak.Set(&userMap, node.GetUser().GetName(), []*v1.Node{node})
} else {
userMap[node.GetUser().GetName()] = append(userMap[node.GetUser().GetName()], node)
}
}
return userMap, nil
}
func (t *HeadscaleInContainer) NodesByName() (map[string]*v1.Node, error) {
nodes, err := t.ListNodes()
if err != nil {
return nil, err
}
var nameMap map[string]*v1.Node
for _, node := range nodes {
mak.Set(&nameMap, node.GetName(), node)
}
return nameMap, nil
}
2025-02-01 09:16:51 +00:00
// ListUsers returns a list of users from Headscale.
func (t *HeadscaleInContainer) ListUsers() ([]*v1.User, error) {
command := []string{binHeadscale, "users", "list", flagOutput, "json"}
result, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return nil, fmt.Errorf("executing list node command: %w", err)
}
2025-02-01 09:16:51 +00:00
var users []*v1.User
2026-02-06 21:45:32 +01:00
2025-02-01 09:16:51 +00:00
err = json.Unmarshal([]byte(result), &users)
if err != nil {
return nil, fmt.Errorf("unmarshalling nodes: %w", err)
}
2025-02-01 09:16:51 +00:00
return users, nil
}
// MapUsers returns a map of users from Headscale. It is keyed by the
// user name.
func (t *HeadscaleInContainer) MapUsers() (map[string]*v1.User, error) {
users, err := t.ListUsers()
if err != nil {
return nil, err
}
var userMap map[string]*v1.User
for _, user := range users {
mak.Set(&userMap, user.GetName(), user)
}
return userMap, nil
}
// DeleteUser deletes a user from the Headscale instance.
func (t *HeadscaleInContainer) DeleteUser(userID uint64) error {
command := []string{
binHeadscale,
"users",
"delete",
"--identifier",
strconv.FormatUint(userID, 10),
"--force",
flagOutput,
"json",
}
_, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return fmt.Errorf("executing delete user command: %w", err)
}
return nil
}
func (h *HeadscaleInContainer) SetPolicy(pol *policyv2.Policy) error {
err := h.writePolicy(pol)
if err != nil {
return fmt.Errorf("writing policy file: %w", err)
}
switch h.policyMode {
case types.PolicyModeDB:
err := h.reloadDatabasePolicy()
if err != nil {
return fmt.Errorf("reloading database policy: %w", err)
}
case types.PolicyModeFile:
err := h.Reload()
if err != nil {
return fmt.Errorf("reloading policy file: %w", err)
}
default:
panic("policy mode is not valid: " + h.policyMode)
}
return nil
}
func (h *HeadscaleInContainer) reloadDatabasePolicy() error {
_, err := h.Execute(
[]string{
binHeadscale,
"policy",
"set",
"-f",
aclPolicyPath,
},
)
if err != nil {
return fmt.Errorf("setting policy with db command: %w", err)
}
return nil
}
func (h *HeadscaleInContainer) writePolicy(pol *policyv2.Policy) error {
pBytes, err := json.Marshal(pol)
if err != nil {
return fmt.Errorf("marshalling policy: %w", err)
}
err = h.WriteFile(aclPolicyPath, pBytes)
if err != nil {
return fmt.Errorf("writing policy to headscale container: %w", err)
}
return nil
}
func (h *HeadscaleInContainer) PID() (int, error) {
// Use pidof to find the headscale process, which is more reliable than grep
// as it only looks for the actual binary name, not processes that contain
// "headscale" in their command line (like the dlv debugger).
output, err := h.Execute([]string{"pidof", binHeadscale})
if err != nil {
// pidof returns exit code 1 when no process is found
return 0, os.ErrNotExist
}
// pidof returns space-separated PIDs on a single line
pidStrs := strings.Fields(strings.TrimSpace(output))
if len(pidStrs) == 0 {
return 0, os.ErrNotExist
}
pids := make([]int, 0, len(pidStrs))
for _, pidStr := range pidStrs {
pidInt, err := strconv.Atoi(pidStr)
if err != nil {
return 0, fmt.Errorf("parsing PID %q: %w", pidStr, err)
}
// We dont care about the root pid for the container
if pidInt == 1 {
continue
}
2026-02-06 21:45:32 +01:00
pids = append(pids, pidInt)
}
switch len(pids) {
case 0:
return 0, os.ErrNotExist
case 1:
return pids[0], nil
default:
// If we still have multiple PIDs, return the first one as a fallback
// This can happen in edge cases during startup/shutdown
return pids[0], nil
}
}
// Reload sends a SIGHUP to the headscale process to reload internals,
// for example Policy from file.
func (h *HeadscaleInContainer) Reload() error {
pid, err := h.PID()
if err != nil {
return fmt.Errorf("getting headscale PID: %w", err)
}
_, err = h.Execute([]string{"kill", "-HUP", strconv.Itoa(pid)})
if err != nil {
return fmt.Errorf("reloading headscale with HUP: %w", err)
}
return nil
}
// ApproveRoutes approves routes for a node.
func (t *HeadscaleInContainer) ApproveRoutes(id uint64, routes []netip.Prefix) (*v1.Node, error) {
command := []string{
binHeadscale, "nodes", "approve-routes",
flagOutput, "json",
"--identifier", strconv.FormatUint(id, 10),
"--routes=" + strings.Join(util.PrefixesToString(routes), ","),
}
result, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return nil, fmt.Errorf(
"executing approve routes command (node %d, routes %v): %w",
id,
routes,
err,
)
}
var node *v1.Node
2026-02-06 21:45:32 +01:00
err = json.Unmarshal([]byte(result), &node)
if err != nil {
return nil, fmt.Errorf("unmarshalling node response: %q, error: %w", result, err)
}
return node, nil
}
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
// SetNodeTags sets tags on a node via the headscale CLI.
// This simulates what the Tailscale admin console UI does - it calls the headscale
// SetTags API which is exposed via the CLI command: headscale nodes tag -i <id> -t <tags>.
func (t *HeadscaleInContainer) SetNodeTags(nodeID uint64, tags []string) error {
command := []string{
binHeadscale, "nodes", "tag",
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
"--identifier", strconv.FormatUint(nodeID, 10),
flagOutput, "json",
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
}
// Add tags - the CLI expects -t flag for each tag or comma-separated
if len(tags) > 0 {
command = append(command, "--tags", strings.Join(tags, ","))
} else {
// Empty tags to clear all tags
command = append(command, "--tags", "")
}
_, _, err := dockertestutil.ExecuteCommand(
t.container,
command,
[]string{},
)
if err != nil {
return fmt.Errorf("executing set tags command (node %d, tags %v): %w", nodeID, tags, err)
tags: process tags on registration, simplify policy (#2931) This PR investigates, adds tests and aims to correctly implement Tailscale's model for how Tags should be accepted, assigned and used to identify nodes in the Tailscale access and ownership model. When evaluating in Headscale's policy, Tags are now only checked against a nodes "tags" list, which defines the source of truth for all tags for a given node. This simplifies the code for dealing with tags greatly, and should help us have less access bugs related to nodes belonging to tags or users. A node can either be owned by a user, or a tag. Next, to ensure the tags list on the node is correctly implemented, we first add tests for every registration scenario and combination of user, pre auth key and pre auth key with tags with the same registration expectation as observed by trying them all with the Tailscale control server. This should ensure that we implement the correct behaviour and that it does not change or break over time. Lastly, the missing parts of the auth has been added, or changed in the cases where it was wrong. This has in large parts allowed us to delete and simplify a lot of code. Now, tags can only be changed when a node authenticates or if set via the CLI/API. Tags can only be fully overwritten/replaced and any use of either auth or CLI will replace the current set if different. A user owned device can be converted to a tagged device, but it cannot be changed back. A tagged device can never remove the last tag either, it has to have a minimum of one.
2025-12-08 18:51:07 +01:00
}
return nil
}
// WriteFile save file inside the Headscale container.
func (t *HeadscaleInContainer) WriteFile(path string, data []byte) error {
return integrationutil.WriteFileToContainer(t.pool, t.container, path, data)
}
// FetchPath gets a path from inside the Headscale container and returns a tar
// file as byte array.
func (t *HeadscaleInContainer) FetchPath(path string) ([]byte, error) {
return integrationutil.FetchPathFromContainer(t.pool, t.container, path)
}
func (t *HeadscaleInContainer) SendInterrupt() error {
pid, err := t.Execute([]string{"pidof", binHeadscale})
if err != nil {
return err
}
_, err = t.Execute([]string{"kill", "-2", strings.Trim(pid, "'\n")})
if err != nil {
return err
}
return nil
}
2025-08-27 17:09:13 +02:00
func (t *HeadscaleInContainer) GetAllMapReponses() (map[types.NodeID][]tailcfg.MapResponse, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/mapresponses",
2025-08-27 17:09:13 +02:00
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching mapresponses from debug endpoint: %w", err)
}
var res map[types.NodeID][]tailcfg.MapResponse
2026-02-06 21:45:32 +01:00
if err := json.Unmarshal([]byte(result), &res); err != nil { //nolint:noinlineerr
2025-08-27 17:09:13 +02:00
return nil, fmt.Errorf("decoding routes response: %w", err)
}
return res, nil
}
// PrimaryRoutes fetches the primary routes from the debug endpoint.
func (t *HeadscaleInContainer) PrimaryRoutes() (*types.DebugRoutes, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/routes",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching routes from debug endpoint: %w", err)
}
var debugRoutes types.DebugRoutes
2026-02-06 21:45:32 +01:00
if err := json.Unmarshal([]byte(result), &debugRoutes); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding routes response: %w", err)
}
return &debugRoutes, nil
}
// DebugBatcher fetches the batcher debug information from the debug endpoint.
func (t *HeadscaleInContainer) DebugBatcher() (*hscontrol.DebugBatcherInfo, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/batcher",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching batcher debug info: %w", err)
}
var debugInfo hscontrol.DebugBatcherInfo
2026-02-06 21:45:32 +01:00
if err := json.Unmarshal([]byte(result), &debugInfo); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding batcher debug response: %w", err)
}
return &debugInfo, nil
}
// DebugNodeStore fetches the [state.NodeStore] data from the debug endpoint.
func (t *HeadscaleInContainer) DebugNodeStore() (map[types.NodeID]types.Node, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/nodestore",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching nodestore debug info: %w", err)
}
var nodeStore map[types.NodeID]types.Node
2026-02-06 21:45:32 +01:00
if err := json.Unmarshal([]byte(result), &nodeStore); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding nodestore debug response: %w", err)
}
return nodeStore, nil
}
// DebugFilter fetches the current filter rules from the debug endpoint.
func (t *HeadscaleInContainer) DebugFilter() ([]tailcfg.FilterRule, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "-H", acceptJSON, "http://localhost:9090/debug/filter",
}
result, err := t.Execute(command)
if err != nil {
return nil, fmt.Errorf("fetching filter from debug endpoint: %w", err)
}
var filterRules []tailcfg.FilterRule
2026-02-06 21:45:32 +01:00
if err := json.Unmarshal([]byte(result), &filterRules); err != nil { //nolint:noinlineerr
return nil, fmt.Errorf("decoding filter response: %w", err)
}
return filterRules, nil
}
// DebugPolicy fetches the current policy from the debug endpoint.
func (t *HeadscaleInContainer) DebugPolicy() (string, error) {
// Execute curl inside the container to access the debug endpoint locally
command := []string{
"curl", "-s", "http://localhost:9090/debug/policy",
}
result, err := t.Execute(command)
if err != nil {
return "", fmt.Errorf("fetching policy from debug endpoint: %w", err)
}
return result, nil
}