mapper: close stale map channels after send timeouts
When the batcher timed out sending to a node, it removed the channel from multiChannelNodeConn but left the old serveLongPoll goroutine running on that channel. That left a live stale session behind: it no longer received new updates, but it could still keep the stream open and block shutdown. Close the pruned channel when stale-send cleanup removes it so the old map session exits after draining any buffered update.
This commit is contained in:
parent
b81d6c734d
commit
3daf45e88a
2 changed files with 214 additions and 8 deletions
|
|
@ -556,13 +556,32 @@ func (mc *multiChannelNodeConn) close() {
|
|||
defer mc.mutex.Unlock()
|
||||
|
||||
for _, conn := range mc.connections {
|
||||
// Mark as closed before closing the channel to prevent
|
||||
// send on closed channel panics from concurrent workers
|
||||
conn.closed.Store(true)
|
||||
mc.closeConnection(conn)
|
||||
}
|
||||
}
|
||||
|
||||
// closeConnection closes connection channel at most once, even if multiple cleanup
|
||||
// paths race to tear the same session down.
|
||||
func (mc *multiChannelNodeConn) closeConnection(conn *connectionEntry) {
|
||||
if conn.closed.CompareAndSwap(false, true) {
|
||||
close(conn.c)
|
||||
}
|
||||
}
|
||||
|
||||
// removeConnectionAtIndexLocked removes the active connection at index.
|
||||
// If closeChannel is true, it also closes that session's map-response channel.
|
||||
// Caller must hold mc.mutex.
|
||||
func (mc *multiChannelNodeConn) removeConnectionAtIndexLocked(i int, closeChannel bool) *connectionEntry {
|
||||
conn := mc.connections[i]
|
||||
mc.connections = append(mc.connections[:i], mc.connections[i+1:]...)
|
||||
|
||||
if closeChannel {
|
||||
mc.closeConnection(conn)
|
||||
}
|
||||
|
||||
return conn
|
||||
}
|
||||
|
||||
// addConnection adds a new connection.
|
||||
func (mc *multiChannelNodeConn) addConnection(entry *connectionEntry) {
|
||||
mutexWaitStart := time.Now()
|
||||
|
|
@ -590,8 +609,7 @@ func (mc *multiChannelNodeConn) removeConnectionByChannel(c chan<- *tailcfg.MapR
|
|||
|
||||
for i, entry := range mc.connections {
|
||||
if entry.c == c {
|
||||
// Remove this connection
|
||||
mc.connections = append(mc.connections[:i], mc.connections[i+1:]...)
|
||||
mc.removeConnectionAtIndexLocked(i, false)
|
||||
mc.log.Debug().Caller().Str(zf.Chan, fmt.Sprintf("%p", c)).
|
||||
Int("remaining_connections", len(mc.connections)).
|
||||
Msg("successfully removed connection")
|
||||
|
|
@ -673,10 +691,10 @@ func (mc *multiChannelNodeConn) send(data *tailcfg.MapResponse) error {
|
|||
// Remove failed connections (in reverse order to maintain indices)
|
||||
for i := len(failedConnections) - 1; i >= 0; i-- {
|
||||
idx := failedConnections[i]
|
||||
entry := mc.removeConnectionAtIndexLocked(idx, true)
|
||||
mc.log.Debug().Caller().
|
||||
Str(zf.ConnID, mc.connections[idx].id).
|
||||
Msg("send: removing failed connection")
|
||||
mc.connections = append(mc.connections[:idx], mc.connections[idx+1:]...)
|
||||
Str(zf.ConnID, entry.id).
|
||||
Msg("send: removed failed connection")
|
||||
}
|
||||
|
||||
mc.updateCount.Add(1)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue