hscontrol/poll,state: fix grace period disconnect TOCTOU race
When a node disconnects, serveLongPoll defers a cleanup that starts a grace period goroutine. This goroutine polls batcher.IsConnected() and, if the node has not reconnected within ~10 seconds, calls state.Disconnect() to mark it offline. A TOCTOU race exists: the node can reconnect (calling Connect()) between the IsConnected check and the Disconnect() call, causing the stale Disconnect() to overwrite the new session's online status. Fix with a monotonic per-node generation counter: - State.Connect() increments the counter and returns the current generation alongside the change list. - State.Disconnect() accepts the generation from the caller and rejects the call if a newer generation exists, making stale disconnects from old sessions a no-op. - serveLongPoll captures the generation at Connect() time and passes it to Disconnect() in the deferred cleanup. - RemoveNode's return value is now checked: if another session already owns the batcher slot (reconnect happened), the old session skips the grace period entirely. Update batcher_test.go to track per-node connect generations and pass them through to Disconnect(), matching production behavior. Fixes the following test failures: - server_state_online_after_reconnect_within_grace - update_history_no_false_offline - nodestore_correct_after_rapid_reconnect - rapid_reconnect_peer_never_sees_offline
This commit is contained in:
parent
00c41b6422
commit
b09af3846b
3 changed files with 100 additions and 11 deletions
|
|
@ -11,6 +11,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/juanfont/headscale/hscontrol/types"
|
||||
"github.com/juanfont/headscale/hscontrol/types/change"
|
||||
"github.com/juanfont/headscale/hscontrol/util"
|
||||
"github.com/juanfont/headscale/hscontrol/util/zlog/zf"
|
||||
"github.com/rs/zerolog"
|
||||
|
|
@ -147,11 +148,24 @@ func (m *mapSession) serveLongPoll() {
|
|||
|
||||
m.log.Trace().Caller().Msg("long poll session started")
|
||||
|
||||
// connectGen is set by Connect() below and captured by the deferred cleanup closure.
|
||||
// It allows Disconnect() to reject stale calls from old sessions — if a newer session
|
||||
// has called Connect() (incrementing the generation), the old session's Disconnect()
|
||||
// sees a mismatched generation and becomes a no-op.
|
||||
var connectGen uint64
|
||||
|
||||
// Clean up the session when the client disconnects
|
||||
defer func() {
|
||||
m.stopFromBatcher()
|
||||
|
||||
_ = m.h.mapBatcher.RemoveNode(m.node.ID, m.ch)
|
||||
stillConnected := m.h.mapBatcher.RemoveNode(m.node.ID, m.ch)
|
||||
|
||||
// If another session already exists for this node (reconnect
|
||||
// happened before this cleanup ran), skip the grace period
|
||||
// entirely — the node is not actually disconnecting.
|
||||
if stillConnected {
|
||||
return
|
||||
}
|
||||
|
||||
// When a node disconnects, it might rapidly reconnect (e.g. mobile clients, network weather).
|
||||
// Instead of immediately marking the node as offline, we wait a few seconds to see if it reconnects.
|
||||
|
|
@ -176,7 +190,11 @@ func (m *mapSession) serveLongPoll() {
|
|||
}
|
||||
|
||||
if disconnected {
|
||||
disconnectChanges, err := m.h.state.Disconnect(m.node.ID)
|
||||
// Pass the generation from our Connect() call. If a newer session has
|
||||
// connected since (bumping the generation), Disconnect() will detect
|
||||
// the mismatch and skip the state update, preventing the race where
|
||||
// an old grace period goroutine overwrites a newer session's online status.
|
||||
disconnectChanges, err := m.h.state.Disconnect(m.node.ID, connectGen)
|
||||
if err != nil {
|
||||
m.log.Error().Caller().Err(err).Msg("failed to disconnect node")
|
||||
}
|
||||
|
|
@ -215,7 +233,9 @@ func (m *mapSession) serveLongPoll() {
|
|||
// 2. Connect: marks the node online and recalculates primary routes based on the updated state
|
||||
// While this results in two notifications, it ensures route data is synchronized before
|
||||
// primary route selection occurs, which is critical for proper HA subnet router failover.
|
||||
connectChanges := m.h.state.Connect(m.node.ID)
|
||||
var connectChanges []change.Change
|
||||
|
||||
connectChanges, connectGen = m.h.state.Connect(m.node.ID)
|
||||
|
||||
m.log.Info().Caller().Str(zf.Chan, fmt.Sprintf("%p", m.ch)).Msg("node has connected")
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue