fix(mobile): reset reconnect attempts only after the E2EE handshake completes (#11465)

ws.onopen zeroed reconnectAttempt before the handshake, so any endpoint
that accepted the socket but never authenticated pinned the counter at
0-1: no escalation gate could fire, backoff never grew, and every screen
showed "Connecting…" forever (issue #10119). Reset the counter on
e2ee_authenticated instead, and make classifyConnection apply the
warning/unreachable gates during connecting/handshaking so an escalated
verdict latches through redials.
This commit is contained in:
Brennan Benson 2026-07-30 12:59:01 -07:00 committed by GitHub
parent 292626eebb
commit d413dfb424
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 442 additions and 13 deletions

View File

@ -0,0 +1,246 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { connect } from './rpc-client'
import { classifyConnection, verdictDisplayLabel } from './connection-health'
// Issue #10119 — "mobile client fails to connect over cellular and remains stuck
// at connecting". Each scenario below is a way a cellular link fails and what the
// user is shown while it does. On the parent commit ws.onopen reset
// reconnectAttempt before the E2EE handshake completed and classifyConnection
// reverted to "Connecting…" on every redial, so no escalation assertion here
// could pass: the label looped "Connecting…" forever.
vi.mock('./e2ee', () => ({
generateKeyPair: () => ({ publicKey: new Uint8Array(32), secretKey: new Uint8Array(32) }),
deriveSharedKey: () => new Uint8Array(32),
publicKeyFromBase64: () => new Uint8Array(32),
publicKeyToBase64: () => 'client-public-key',
encrypt: (plaintext: string) => plaintext,
decrypt: (raw: string) => raw,
decryptBytes: (bytes: Uint8Array) => bytes
}))
type CarrierBehavior =
// Carrier silently drops the SYN to a LAN/CGNAT destination: the socket sits
// CONNECTING until the client's 12s connect timeout fires.
| { kind: 'blackhole' }
// Carrier answers with an RST.
| { kind: 'refused' }
// Endpoint completes the WS upgrade, then never speaks (relay accepted the
// socket but no desktop joined the session).
| { kind: 'open-then-silent' }
// Endpoint is healthy but its e2ee_ready lands after readyAfterMs, which a
// high-latency / lossy cellular link readily pushes past the 5s budget.
| { kind: 'slow-handshake'; readyAfterMs: number }
let carrier: CarrierBehavior = { kind: 'blackhole' }
const sockets: CarrierWebSocket[] = []
class CarrierWebSocket {
static CONNECTING = 0
// Why: sendEncrypted compares readyState against WebSocket.OPEN — the mock
// must define it or the e2ee_auth write silently fails.
static OPEN = 1
static CLOSED = 3
readonly CONNECTING = 0
readyState = 0
onopen: (() => void) | null = null
onclose: ((event?: unknown) => void) | null = null
onmessage: ((event: { data: unknown }) => void) | null = null
onerror: ((event?: unknown) => void) | null = null
constructor(readonly endpoint: string) {
sockets.push(this)
if (carrier.kind === 'refused') {
setTimeout(() => this.fail(), 1)
return
}
if (carrier.kind === 'open-then-silent' || carrier.kind === 'slow-handshake') {
setTimeout(() => {
this.readyState = 1
this.onopen?.()
}, 200)
}
}
send(payload: string): void {
if (carrier.kind !== 'slow-handshake') {
return
}
if (payload.includes('e2ee_hello')) {
const delay = carrier.readyAfterMs
setTimeout(() => {
if (this.readyState === 1) {
this.onmessage?.({ data: JSON.stringify({ type: 'e2ee_ready' }) })
}
}, delay)
return
}
if (payload.includes('e2ee_auth')) {
setTimeout(() => {
if (this.readyState === 1) {
this.onmessage?.({ data: JSON.stringify({ type: 'e2ee_authenticated' }) })
}
}, 10)
}
}
close(): void {
if (this.readyState === 3) {
return
}
this.fail()
}
private fail(): void {
this.readyState = 3
this.onerror?.({ message: 'network error' })
this.onclose?.({ code: 1006, wasClean: false })
}
}
const originalWebSocket = globalThis.WebSocket
type Sample = { atMs: number; state: string; attempts: number; label: string }
// Walks simulated time recording what every screen would render, since
// classifyConnection is the single label source for home, host, tasks and session.
function observe(endpoint: string, durationMs: number): Sample[] {
const client = connect(endpoint, 'device-token', 'server-public-key')
const samples: Sample[] = []
const stepMs = 250
for (let atMs = 0; atMs <= durationMs; atMs += stepMs) {
const state = client.getState()
const attempts = client.getReconnectAttempt()
samples.push({
atMs,
state,
attempts,
label: verdictDisplayLabel(
classifyConnection({
state,
reconnectAttempts: attempts,
lastConnectedAt: client.getLastConnectedAt(),
endpoint,
nowMs: Date.now()
})
)
})
vi.advanceTimersByTime(stepMs)
}
client.close()
return samples
}
function isEscalated(label: string): boolean {
return label !== 'Connecting…' && label !== 'Reconnecting…'
}
function labelsAfterFirstEscalation(samples: Sample[]): Sample[] {
const first = samples.findIndex((s) => isEscalated(s.label))
expect(first).toBeGreaterThan(0)
return samples.slice(first)
}
describe('issue #10119 — what a phone shows while it cannot reach the desktop', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.spyOn(console, 'log').mockImplementation(() => {})
sockets.length = 0
// @ts-expect-error test double for the RN global
globalThis.WebSocket = CarrierWebSocket
})
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
globalThis.WebSocket = originalWebSocket
})
it('escalates and stays escalated when the handshake never fits the 5s budget', () => {
carrier = { kind: 'slow-handshake', readyAfterMs: 6_000 }
const samples = observe('ws://192.168.0.56:6769', 900_000)
// 15 minutes of failure against a desktop that is up and answering.
expect(samples.some((s) => s.state === 'handshaking')).toBe(true)
expect(samples.some((s) => s.state === 'connected')).toBe(false)
// The failure counter survives ws.onopen, so both connection-health gates
// (warning >= 3, unreachable >= 12) are reachable.
expect(Math.max(...samples.map((s) => s.attempts))).toBeGreaterThanOrEqual(12)
expect(samples.some((s) => s.label === "Can't connect")).toBe(true)
expect(samples.some((s) => s.label === "Can't reach desktop")).toBe(true)
// Once escalated, no later redial reverts the label to "Connecting…".
const after = labelsAfterFirstEscalation(samples)
expect(after.every((s) => s.label !== 'Connecting…')).toBe(true)
// Growing backoff: the pinned counter burned a socket every ~5.6s (~160 in
// 15 min); with the tiered delays plus the 90s trickle it stays modest.
expect(sockets.length).toBeGreaterThanOrEqual(5)
expect(sockets.length).toBeLessThan(30)
})
it('escalates when the endpoint accepts the socket and then says nothing', () => {
carrier = { kind: 'open-then-silent' }
const samples = observe('ws://relay.example:443', 900_000)
expect(Math.max(...samples.map((s) => s.attempts))).toBeGreaterThanOrEqual(12)
expect(samples.some((s) => s.label === "Can't reach desktop")).toBe(true)
const after = labelsAfterFirstEscalation(samples)
expect(after.every((s) => s.label !== 'Connecting…')).toBe(true)
})
it('latches "Can\'t connect" on a blackholed LAN endpoint instead of reverting each dial', () => {
carrier = { kind: 'blackhole' }
const samples = observe('ws://192.168.0.56:6769', 120_000)
// Three 12s connect timeouts plus backoff: first escalation lands ≈ 37.5s.
const first = samples.find((s) => isEscalated(s.label))
expect(first?.atMs ?? Infinity).toBeLessThanOrEqual(40_000)
// Every later 12s dial window used to flip the label back to "Connecting…".
const after = labelsAfterFirstEscalation(samples)
expect(after.every((s) => s.label !== 'Connecting…')).toBe(true)
})
it('holds "Can\'t reach desktop" through every trickle dial once past the give-up cap', () => {
carrier = { kind: 'blackhole' }
const samples = observe('ws://192.168.0.56:6769', 900_000)
const firstUnreachable = samples.findIndex((s) => s.label === "Can't reach desktop")
expect(firstUnreachable).toBeGreaterThan(0)
// The loop has given up internally (attempts held at the cap); the label
// must say so through the 12s of every 90s trickle dial, not just between them.
const after = samples.slice(firstUnreachable)
expect(after.every((s) => s.attempts >= 12)).toBe(true)
expect(after.every((s) => s.label === "Can't reach desktop")).toBe(true)
})
it('resets the failure counter only once a handshake actually completes', () => {
carrier = { kind: 'slow-handshake', readyAfterMs: 6_000 }
const client = connect('ws://192.168.0.56:6769', 'device-token', 'server-public-key')
vi.advanceTimersByTime(40_000)
expect(client.getState()).not.toBe('connected')
expect(client.getReconnectAttempt()).toBeGreaterThanOrEqual(3)
// The link heals: the same desktop now answers inside the budget.
carrier = { kind: 'slow-handshake', readyAfterMs: 50 }
vi.advanceTimersByTime(20_000)
expect(client.getState()).toBe('connected')
expect(client.getReconnectAttempt()).toBe(0)
client.close()
})
it('contrast: an endpoint that RSTs escalates the same way', () => {
carrier = { kind: 'refused' }
const samples = observe('ws://192.168.0.56:6769', 60_000)
expect(samples.some((s) => s.label === "Can't connect")).toBe(true)
const after = labelsAfterFirstEscalation(samples)
expect(after.every((s) => s.label !== 'Connecting…')).toBe(true)
})
})

View File

@ -0,0 +1,136 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { WebSocketServer, type WebSocket as ServerWebSocket } from 'ws'
import { connect } from './rpc-client'
import { classifyConnection, verdictDisplayLabel } from './connection-health'
// Issue #10119 verification over a REAL socket, real timers, real WebSocket
// upgrade — guards the fake-timer suite in
// cellular-connecting-label-stall.test.ts against mock drift.
//
// The desktop is up and listening. The only thing wrong is that its E2EE
// handshake reply lands later than HANDSHAKE_TIMEOUT_MS (5s) — the condition a
// high-latency / lossy cellular link produces on a handshake that needs two
// round trips (e2ee_hello → e2ee_ready → e2ee_auth → e2ee_authenticated).
//
// The client must escalate past "Connecting…" and stay escalated. On the parent
// commit ws.onopen reset reconnectAttempt to 0 before the handshake succeeded,
// so getReconnectAttempt() never passed the connection-health gates and the
// label looped "Connecting…" forever.
//
// Opt-in like rpc-client-live-recovery.test.ts — needs ~22s wall-clock:
// ORCA_MOBILE_LIVE_REPRO=1 pnpm vitest run src/transport/cellular-handshake-stall-real-socket.test.ts
vi.mock('./e2ee', () => ({
generateKeyPair: () => ({ publicKey: new Uint8Array(32), secretKey: new Uint8Array(32) }),
deriveSharedKey: () => new Uint8Array(32),
publicKeyFromBase64: () => new Uint8Array(32),
publicKeyToBase64: () => 'client-public-key',
encrypt: (plaintext: string) => plaintext,
decrypt: (raw: string) => raw,
decryptBytes: (bytes: Uint8Array) => bytes
}))
const RUN_LIVE =
process.env.ORCA_MOBILE_LIVE_REPRO === '1' || !!process.env.ORCA_MOBILE_LIVE_REPRO_FULL
// Long enough for three handshake-timeout cycles (≈17s) plus CI scheduling slack.
const OBSERVE_MS = 22_000
const SAMPLE_MS = 200
let server: WebSocketServer | null = null
const serverSockets: ServerWebSocket[] = []
// Completes the WebSocket upgrade, then answers e2ee_hello later than the
// client's 5s handshake budget allows.
async function startSlowHandshakeDesktop(replyAfterMs: number): Promise<number> {
const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 })
server = wss
wss.on('connection', (socket) => {
serverSockets.push(socket)
socket.on('message', (data) => {
const text = data.toString()
if (!text.includes('e2ee_hello')) {
return
}
setTimeout(() => {
if (socket.readyState === socket.OPEN) {
socket.send(JSON.stringify({ type: 'e2ee_ready', publicKeyB64: 'server-public-key' }))
}
}, replyAfterMs)
})
})
await new Promise<void>((resolve) => wss.once('listening', resolve))
const address = wss.address()
if (typeof address === 'string' || address === null) {
throw new Error('expected a TCP address')
}
return address.port
}
describe.runIf(RUN_LIVE)('issue #10119 — real socket, handshake slower than the 5s budget', () => {
beforeEach(() => {
serverSockets.length = 0
vi.spyOn(console, 'log').mockImplementation(() => {})
})
afterEach(async () => {
vi.restoreAllMocks()
for (const socket of serverSockets) {
socket.terminate()
}
const wss = server
server = null
if (wss) {
await new Promise<void>((resolve) => wss.close(() => resolve()))
}
})
it('escalates and latches against a live desktop that answers 1s too late', async () => {
const port = await startSlowHandshakeDesktop(6_000)
const endpoint = `ws://127.0.0.1:${port}`
const client = connect(endpoint, 'device-token', 'server-public-key')
const labels: string[] = []
const states: string[] = []
let maxAttempts = 0
const started = Date.now()
while (Date.now() - started < OBSERVE_MS) {
const state = client.getState()
const attempts = client.getReconnectAttempt()
maxAttempts = Math.max(maxAttempts, attempts)
states.push(state)
labels.push(
verdictDisplayLabel(
classifyConnection({
state,
reconnectAttempts: attempts,
lastConnectedAt: client.getLastConnectedAt(),
endpoint
})
)
)
await new Promise((resolve) => setTimeout(resolve, SAMPLE_MS))
}
client.close()
// The dial loop really did run and really did keep failing.
expect(serverSockets.length).toBeGreaterThanOrEqual(2)
expect(states).toContain('handshaking')
expect(states).not.toContain('connected')
// The failure counter survives ws.onopen, so the warning gate (>= 3) fires.
expect(maxAttempts).toBeGreaterThanOrEqual(3)
const firstEscalated = labels.findIndex((l) => l !== 'Connecting…' && l !== 'Reconnecting…')
expect(firstEscalated).toBeGreaterThan(0)
// Once escalated, later redials never revert the label to "Connecting…".
expect(labels.slice(firstEscalated).every((l) => l !== 'Connecting…')).toBe(true)
}, 60_000)
})
// Why: vitest fails a file with zero tests; keep a sentinel for default runs.
describe.runIf(!RUN_LIVE)('real-socket handshake stall (skipped)', () => {
it('is opt-in via ORCA_MOBILE_LIVE_REPRO=1', () => {
expect(true).toBe(true)
})
})

View File

@ -71,6 +71,45 @@ describe('classifyConnection Tailscale hint', () => {
})
})
// Issue #10119: every redial re-enters 'connecting', which used to revert an
// escalated verdict to "Connecting…" for the whole dial window — on a loop that
// had already failed for minutes, the user mostly saw the reassuring label.
describe('classifyConnection while dialing (issue #10119)', () => {
const base = { lastConnectedAt: null, nowMs: 1_000_000 }
it('keeps the warning verdict through a redial instead of reverting to Connecting…', () => {
for (const state of ['connecting', 'handshaking'] as const) {
const verdict = classifyConnection({ ...base, state, reconnectAttempts: 3 })
expect(verdict).toMatchObject({ kind: 'warning', label: "Can't connect" })
}
})
it('keeps the unreachable verdict through a trickle dial', () => {
const verdict = classifyConnection({ ...base, state: 'connecting', reconnectAttempts: 12 })
expect(verdict).toMatchObject({ kind: 'unreachable', reason: 'never-connected' })
})
it('applies the stale heuristic while dialing too', () => {
const verdict = classifyConnection({
state: 'handshaking',
reconnectAttempts: 12,
lastConnectedAt: 900_000,
nowMs: 1_000_000
})
expect(verdict).toMatchObject({ kind: 'unreachable', reason: 'stale' })
})
it('still shows Connecting… before any failures', () => {
const verdict = classifyConnection({ ...base, state: 'connecting', reconnectAttempts: 0 })
expect(verdict).toEqual({ kind: 'normal', label: 'Connecting…' })
})
it('still shows Connecting… below the warning gate', () => {
const verdict = classifyConnection({ ...base, state: 'handshaking', reconnectAttempts: 2 })
expect(verdict).toEqual({ kind: 'normal', label: 'Connecting…' })
})
})
describe('verdictDisplayLabel', () => {
it('appends the hint to warning and unreachable labels', () => {
expect(

View File

@ -61,19 +61,18 @@ export function classifyConnection(args: {
return { kind: 'auth-failed', label: 'Pairing invalid — re-pair with your desktop' }
}
// Connected / connecting / handshaking are normal.
if (state === 'connected') {
return { kind: 'normal', label: 'Connected' }
}
if (state === 'connecting' || state === 'handshaking') {
return { kind: 'normal', label: 'Connecting…' }
}
if (state === 'disconnected') {
return { kind: 'normal', label: 'Disconnected' }
}
// state === 'reconnecting' from here.
// connecting / handshaking / reconnecting from here. The gates apply to all
// three: every redial re-enters 'connecting', and letting that revert an
// escalated verdict to "Connecting…" hid the failure loop behind a reassuring
// label for most of each cycle (issue #10119).
if (reconnectAttempts >= UNREACHABLE_ATTEMPTS) {
if (lastConnectedAt == null) {
return {
@ -97,7 +96,7 @@ export function classifyConnection(args: {
return { kind: 'warning', label: "Can't connect", hint }
}
return { kind: 'normal', label: 'Reconnecting…' }
return { kind: 'normal', label: state === 'reconnecting' ? 'Reconnecting…' : 'Connecting…' }
}
// Why: single place that turns a verdict into display text so every screen

View File

@ -89,9 +89,10 @@ describe('unauthorized close-code mapping (silent 4001)', () => {
// A desktop with a regenerated keypair can't send a decryptable e2ee_error —
// the phone only ever sees the 4001 close. Three of those must latch.
// Why 1_000: failed handshakes grow the backoff since issue #10119.
for (let i = 0; i < 3; i++) {
if (i > 0) {
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(1_000)
}
const socket = lastSocket()
socket.open()
@ -130,9 +131,10 @@ describe('unauthorized close-code mapping (silent 4001)', () => {
it('shares one budget between decrypted e2ee_error rejections and 4001 closes', async () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
// Why 1_000: failed handshakes grow the backoff since issue #10119.
for (let i = 0; i < 3; i++) {
if (i > 0) {
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(1_000)
}
const socket = lastSocket()
socket.open()

View File

@ -875,9 +875,11 @@ describe('mobile rpc-client connection timeout', () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
// Three consecutive handshake rejections (AUTH_RETRY_BUDGET = 3).
// Why 1_000: failed handshakes grow the backoff since issue #10119 —
// cycle 2 waits RECONNECT_DELAYS[1].
for (let i = 0; i < 3; i++) {
if (i > 0) {
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(1_000)
}
const socket = mockSockets[mockSockets.length - 1]!
socket.open()
@ -894,16 +896,17 @@ describe('mobile rpc-client connection timeout', () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
// Two rejections, then a clean connect resets the budget...
// Why 1_000: failed handshakes grow the backoff since issue #10119.
for (let i = 0; i < 2; i++) {
if (i > 0) {
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(1_000)
}
const socket = mockSockets[mockSockets.length - 1]!
socket.open()
socket.receive(JSON.stringify({ type: 'e2ee_ready' }))
socket.receive('encrypted:{"type":"e2ee_error","error":{"code":"unauthorized"}}')
}
await vi.advanceTimersByTimeAsync(500)
await vi.advanceTimersByTimeAsync(1_000)
authenticate(mockSockets[mockSockets.length - 1]!)
expect(client.getState()).toBe('connected')

View File

@ -89,7 +89,7 @@ export type RpcClient = {
viewport: { cols: number; rows: number }
) => void
getState: () => ConnectionState
// 0 means never failed (reset on successful open); the UI escalates "Reconnecting…" to "Can't connect" past a threshold.
// 0 means never failed (reset once the handshake authenticates); the UI escalates "Reconnecting…" to "Can't connect" past a threshold.
getReconnectAttempt: () => number
// Last 'connected' timestamp (ms epoch); null = never connected. Lets the UI tell "never reachable" from "transient blip".
getLastConnectedAt: () => number | null
@ -220,6 +220,8 @@ export function connect(
})
if (next === 'connected') {
lastConnectedAt = Date.now()
// Why: only a completed E2EE handshake proves the path is healthy (issue #10119).
reconnectAttempt = 0
// Why: a clean handshake proves the token is valid — reset the auth retry budget.
authRejectionCount = 0
for (const waiter of connectWaiters.splice(0)) {
@ -348,7 +350,9 @@ export function connect(
}
console.log('[net] ws.onopen', { attempt: reconnectAttempt })
clearConnectTimer()
reconnectAttempt = 0
// Why: no reconnectAttempt reset here — an open socket isn't a healthy session
// until e2ee_authenticated. Resetting pre-handshake pinned the counter at 0↔1,
// so a handshake-stall loop never escalated past "Connecting…" (issue #10119).
setState('handshaking')
emitLog('success', 'WebSocket open', 'Starting E2EE handshake')