diff --git a/mobile/src/transport/mobile-direct-return-probe.ts b/mobile/src/transport/mobile-direct-return-probe.ts new file mode 100644 index 000000000..dfb0572aa --- /dev/null +++ b/mobile/src/transport/mobile-direct-return-probe.ts @@ -0,0 +1,83 @@ +import { openAuthenticatedDirectEndpoint } from './mobile-direct-endpoint-probe' +import type { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis' +import type { RpcClient } from './rpc-client' +import type { HostProfile } from './types' +import type { MobileConnectionPath } from './stable-logical-rpc-client' + +const DIRECT_PROBE_INTERVAL_MS = 15_000 + +// While the runtime channel rides the relay, periodically probe the direct +// endpoint and migrate back once hysteresis proves it stable. +export class DirectReturnProbe { + private timer: ReturnType | null = null + + constructor( + private readonly deps: { + now: () => number + setTimer: typeof setTimeout + clearTimer: typeof clearTimeout + openDirect: (endpoint: string) => RpcClient + }, + private readonly hooks: { + hysteresis: MobileEndpointHysteresis + host: () => HostProfile + canSchedule: () => boolean + canAttempt: () => boolean + beginOperation: () => void + migrate: (client: RpcClient, path: MobileConnectionPath) => Promise + onDirectMigrated: () => Promise + afterProbe: () => void + } + ) {} + + schedule(delayMs = DIRECT_PROBE_INTERVAL_MS): void { + if (!this.hooks.canSchedule() || this.timer) { + return + } + this.timer = this.deps.setTimer(() => { + this.timer = null + void this.probe() + }, delayMs) + } + + clear(): void { + if (this.timer) { + this.deps.clearTimer(this.timer) + this.timer = null + } + } + + private async probe(): Promise { + if (!this.hooks.canAttempt() || !this.hooks.hysteresis.canProbe(this.deps.now())) { + this.schedule() + return + } + this.hooks.beginOperation() + let successful: Awaited> = null + try { + successful = await openAuthenticatedDirectEndpoint( + this.hooks.host(), + this.deps.openDirect, + 12_000 + ) + if (!successful) { + this.hooks.hysteresis.recordDirectFailure(this.deps.now()) + return + } + if (!this.hooks.hysteresis.recordDirectSuccess(this.deps.now())) { + successful.client.close() + return + } + await this.hooks.migrate(successful.client, successful.path) + successful = null + this.hooks.hysteresis.recordMigration(this.deps.now()) + await this.hooks.onDirectMigrated() + } finally { + successful?.client.close() + // Why: a relay drop or backoff timer can arrive while the probe owns the + // operation mutex; afterProbe releases it and replays deferred recovery. + this.hooks.afterProbe() + this.schedule() + } + } +} diff --git a/mobile/src/transport/mobile-endpoint-lifecycle.ts b/mobile/src/transport/mobile-endpoint-lifecycle.ts index 57881022a..d41898152 100644 --- a/mobile/src/transport/mobile-endpoint-lifecycle.ts +++ b/mobile/src/transport/mobile-endpoint-lifecycle.ts @@ -90,6 +90,7 @@ function createSupervisor( readBundle: readMobileRelayCredentialBundle, writeBundle: writeMobileRelayCredentialBundle, saveHost, + onLog, now: Date.now, randomBytes: ExpoCrypto.getRandomBytes, setTimer: setTimeout, diff --git a/mobile/src/transport/mobile-endpoint-supervisor-contract.ts b/mobile/src/transport/mobile-endpoint-supervisor-contract.ts index d26e0706f..0098de6e0 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor-contract.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor-contract.ts @@ -3,7 +3,7 @@ import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bund import type { MobileRelayRpcSession } from './mobile-relay-rpc-session' import type { resolveMobileRelayEndpoint } from './mobile-relay-resume-director' import type { RpcClient } from './rpc-client' -import type { HostProfile } from './types' +import type { ConnectionLogSink, HostProfile } from './types' export type MobileEndpointSupervisorDependencies = { openDirect: (endpoint: string) => RpcClient @@ -20,4 +20,5 @@ export type MobileEndpointSupervisorDependencies = { randomBytes: (length: number) => Uint8Array setTimer: typeof setTimeout clearTimer: typeof clearTimeout + onLog?: ConnectionLogSink } diff --git a/mobile/src/transport/mobile-endpoint-supervisor.test.ts b/mobile/src/transport/mobile-endpoint-supervisor.test.ts index 3ec05f5ef..e6d0ff8bb 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor.test.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor.test.ts @@ -52,18 +52,21 @@ class FakeRelaySession extends FakeSession implements MobileRelayRpcSession { constructor( state: ConnectionState, private readonly failure: Error | null = null, - private readonly lease = Date.now() + 120_000 + private readonly resumeExpiry = Date.now() + 30 * 24 * 3_600_000 ) { super(state) } - getLeaseExpiresAt = () => this.lease + // Why: production-realistic defaults — fictional fake values hid three + // live defects in this subsystem (latch, churn, int32 timer overflow). + getAttachDeadlineAt = () => Date.now() + 10_000 + getResumeExpiresAt = () => this.resumeExpiry getResumeConfirmation = () => ({ v: 1 as const, reqId: 'confirm-1', currentVersion: 2, acceptedAs: 'current' as const, renewed: true, - resumeExpiresAt: Date.now() + 300_000 + resumeExpiresAt: this.resumeExpiry }) getFailure = () => this.failure } @@ -716,6 +719,7 @@ describe('mobile endpoint supervisor', () => { .mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4408))) const deps = dependencies({ openRelay, + openDirect: vi.fn(() => new FakeSession('disconnected')), resolveRelay: vi.fn(() => resolvePending) }) const supervisor = new MobileEndpointSupervisor(logical, host, deps) @@ -744,12 +748,13 @@ describe('mobile endpoint supervisor', () => { .mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4408))) const deps = dependencies({ openRelay, + openDirect: vi.fn(() => new FakeSession('disconnected')), resolveRelay: vi.fn(() => resolvePending) }) const supervisor = new MobileEndpointSupervisor(logical, host, deps) await supervisor.start() - await vi.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(60_000) await vi.waitFor(() => expect(deps.resolveRelay).toHaveBeenCalledOnce()) supervisor.setForeground(false) finishResolve?.(relay) @@ -785,11 +790,14 @@ describe('mobile endpoint supervisor', () => { .fn() .mockReturnValueOnce(new FakeRelaySession('connected', null, Date.now() + 31_000)) .mockImplementation(() => new FakeRelaySession('disconnected', new RelayOuterError(4404))) - const deps = dependencies({ openRelay }) + const deps = dependencies({ + openRelay, + openDirect: vi.fn(() => new FakeSession('disconnected')) + }) const supervisor = new MobileEndpointSupervisor(logical, host, deps) await supervisor.start() - await vi.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(60_000) expect(openRelay).toHaveBeenCalledTimes(2) await vi.advanceTimersByTimeAsync(5000) @@ -808,12 +816,13 @@ describe('mobile endpoint supervisor', () => { .mockImplementation(() => new FakeRelaySession('connected')) const deps = dependencies({ openRelay, + openDirect: vi.fn(() => new FakeSession('disconnected')), randomBytes: () => new Uint8Array([128, 0]) }) const supervisor = new MobileEndpointSupervisor(logical, host, deps) await supervisor.start() - await vi.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(60_000) expect(openRelay).toHaveBeenCalledTimes(2) // The old relay can outlive its rejected lease replacement, then close separately. @@ -833,12 +842,13 @@ describe('mobile endpoint supervisor', () => { .mockImplementation(() => new FakeRelaySession('connected')) const deps = dependencies({ openRelay, + openDirect: vi.fn(() => new FakeSession('disconnected')), randomBytes: () => new Uint8Array([128, 0]) }) const supervisor = new MobileEndpointSupervisor(logical, host, deps) await supervisor.start() - await vi.advanceTimersByTimeAsync(1000) + await vi.advanceTimersByTimeAsync(60_000) expect(openRelay).toHaveBeenCalledTimes(2) supervisor.setForeground(true) diff --git a/mobile/src/transport/mobile-endpoint-supervisor.ts b/mobile/src/transport/mobile-endpoint-supervisor.ts index c8b3cdbcf..77ffbbde3 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor.ts @@ -1,5 +1,5 @@ -import { openAuthenticatedDirectEndpoint } from './mobile-direct-endpoint-probe' import type { MobileEndpointSupervisorDependencies } from './mobile-endpoint-supervisor-contract' +import { DirectReturnProbe } from './mobile-direct-return-probe' import { RelayReconnectController } from './mobile-relay-reconnect-controller' import { RelayLeaseRotationTimer } from './mobile-relay-lease-rotation-timer' import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis' @@ -9,6 +9,8 @@ import { persistRelayHost, toError } from './mobile-endpoint-supervisor-support' +import { selectDialableRelayCredentials } from './mobile-relay-credential-selection' +import { createRelayRecoveryLog, type RelayRecoveryLog } from './mobile-relay-recovery-log' import { applyResumeConfirmation, mobileRelayCredentialNeedsRotation, @@ -20,7 +22,6 @@ import type { HostProfile } from './types' export type { MobileEndpointSupervisorDependencies } from './mobile-endpoint-supervisor-contract' -const DIRECT_PROBE_INTERVAL_MS = 15_000 const DIRECT_OBSERVATION_MS = 30_000 const MINIMUM_DWELL_MS = 60_000 const FAILURE_COOLDOWN_MS = 60_000 @@ -32,11 +33,12 @@ export class MobileEndpointSupervisor { private operationInFlight = false private credentialRotationInFlight = false private relayRotationPending = false - private probeTimer: ReturnType | null = null private unsubscribeState: (() => void) | null = null private readonly hysteresis: MobileEndpointHysteresis private readonly relayReconnect: RelayReconnectController private readonly leaseRotation: RelayLeaseRotationTimer + private readonly logRelay: RelayRecoveryLog + private readonly directProbe: DirectReturnProbe constructor( private readonly logical: StableLogicalRpcClient, @@ -49,24 +51,52 @@ export class MobileEndpointSupervisor { failureCooldownMs: FAILURE_COOLDOWN_MS, minimumDwellMs: MINIMUM_DWELL_MS }) + this.logRelay = createRelayRecoveryLog(dependencies.now, dependencies.onLog) this.relayReconnect = new RelayReconnectController(dependencies, this.recoverRelay.bind(this)) this.leaseRotation = new RelayLeaseRotationTimer(dependencies, () => { this.relayRotationPending = true void this.recoverRelay(true) }) + this.directProbe = new DirectReturnProbe(dependencies, { + hysteresis: this.hysteresis, + host: () => this.host, + canSchedule: () => + !this.stopped && this.foreground && this.logical.getActivePath() === 'relay', + canAttempt: () => !this.stopped && this.foreground && !this.operationInFlight, + beginOperation: () => { + this.operationInFlight = true + }, + migrate: (client, path) => this.logical.migrateTo(client, path), + onDirectMigrated: async () => { + this.leaseRotation.clear() + this.relayRotationPending = false + await this.rotateCredentialIfNeeded(this.relayReconnect.resetForDirectConnection()) + }, + afterProbe: () => { + this.operationInFlight = false + if (this.relayRotationPending || this.logical.getState() !== 'connected') { + void this.recoverRelay(this.relayRotationPending) + } + } + }) } async start(): Promise { this.bundle = await this.dependencies.readBundle(this.host.id).catch(() => null) - if (this.stopped || !this.bundle || !this.host.relay) { + if (this.stopped || !this.host.relay) { return } + if (!this.bundle) { + // Why: a Keychain race at open must not kill relay recovery for the whole + // process lifetime; each recovery attempt re-reads the durable bundle. + this.logRelay('credential bundle unavailable at start; recovery will re-read') + } this.unsubscribeState = this.logical.onStateChange((state) => { if (state === 'connected') { if (this.logical.getActivePath() !== 'relay') { void this.rotateCredentialIfNeeded(this.relayReconnect.resetForDirectConnection()) } - this.scheduleDirectProbe() + this.directProbe.schedule() } else { // Why: the direct client enters reconnecting after its first failed // dial and may never publish disconnected while its retry loop lives. @@ -78,7 +108,7 @@ export class MobileEndpointSupervisor { // are still loading, before the supervisor subscribes to state changes. await this.recoverRelay() } else { - this.scheduleDirectProbe() + this.directProbe.schedule() } } @@ -87,11 +117,11 @@ export class MobileEndpointSupervisor { this.foreground = foreground if (foreground) { this.relayReconnect.handleForeground(this.logical, wasForeground) - this.scheduleDirectProbe(0) + this.directProbe.schedule(0) } else { // Why: background phones must not hold billed relay data splices. this.relayReconnect.suspendActiveRelay(this.logical) - this.clearDirectProbeTimer() + this.directProbe.clear() this.relayReconnect.clear() this.leaseRotation.clear() } @@ -101,7 +131,7 @@ export class MobileEndpointSupervisor { this.stopped = true this.unsubscribeState?.() this.unsubscribeState = null - this.clearDirectProbeTimer() + this.directProbe.clear() this.relayReconnect.clear() this.leaseRotation.clear() } @@ -112,7 +142,6 @@ export class MobileEndpointSupervisor { this.stopped || !this.foreground || this.operationInFlight || - !this.bundle || !this.host.relay || (!forceReplacement && !this.relayReconnect.needsRecovery(this.logical.getState())) ) { @@ -121,23 +150,42 @@ export class MobileEndpointSupervisor { // Why: revival and lease timers can overlap resume failures; one shared cooldown // prevents PEER_DROPPED/LIMIT_EXCEEDED reconnect churn. if (this.relayReconnect.shouldDefer()) { + this.logRelay('recovery deferred by cooldown or gate') return } this.operationInFlight = true let lastError: Error | null = null let retryAfterOperation = false try { - const credentials = this.relayReconnect.eligibleCredentials( - this.bundle.current, - this.bundle.grace - ) - for (const credential of credentials) { + const selection = await selectDialableRelayCredentials({ + bundle: this.bundle, + controller: this.relayReconnect, + readBundle: () => this.dependencies.readBundle(this.host.id), + onAdoptedFresherBundle: () => this.logRelay('adopted fresher durable credential bundle') + }) + this.bundle = selection.bundle + if (selection.credentials.length === 0) { + // Why: "expired" vs "missing" separates a sleep-past-expiry phone + // (needs re-pair or LAN) from a Keychain failure in field reports. + this.logRelay( + selection.bundle + ? 'relay credential expired or rejected; slow reprobe armed' + : 'no relay credential bundle; slow reprobe armed' + ) + this.relayReconnect.armCredentialReprobe() + return + } + for (const credential of selection.credentials) { const result = await this.tryRelayCredential(credential) if (result.ok) { retryAfterOperation = this.logical.getState() !== 'connected' return } lastError = result.error + this.logRelay( + 'relay dial failed', + `${result.error.name}: ${String(result.error.message).slice(0, 80)}` + ) if (this.relayReconnect.shouldTryGraceAfterRelayFailure(result.error)) { // Why: a rejected version stays invalid; retry only the grace credential. this.relayReconnect.recordRejectedCredential(credential.version) @@ -145,12 +193,10 @@ export class MobileEndpointSupervisor { break } } - if (credentials.length > 0) { - // Why: cleanup may happen while a relay dial is awaiting the network; - // record its outcome without recreating a foreground retry timer. - const scheduleRetry = !forceReplacement && this.foreground && !this.stopped - this.relayReconnect.registerFailure(lastError, scheduleRetry) - } + // Why: cleanup may happen while a relay dial is awaiting the network; + // record its outcome without recreating a foreground retry timer. + const scheduleRetry = !forceReplacement && this.foreground && !this.stopped + this.relayReconnect.registerFailure(lastError, scheduleRetry) } finally { this.operationInFlight = false if (forceReplacement && this.relayRotationPending && !this.stopped && this.foreground) { @@ -207,6 +253,7 @@ export class MobileEndpointSupervisor { } this.relayRotationPending = false this.hysteresis.recordMigration(this.dependencies.now()) + this.logRelay('runtime channel migrated to relay') const confirmation = session.getResumeConfirmation() if (confirmation) { this.bundle = applyResumeConfirmation(this.bundle, credential.version, confirmation) @@ -215,71 +262,21 @@ export class MobileEndpointSupervisor { await this.dependencies.writeBundle(this.bundle).catch(() => {}) } // Why: async persistence can finish after stop/background; never recreate a stale timer. + // Why: rotate against the resume credential's expiry, never the hello's + // leaseExpiresAt — that field is the cell's ~10s attach-reservation + // deadline, and using it forced a session replacement every second. this.leaseRotation.scheduleFromLease( - this.stopped || !this.foreground ? null : session.getLeaseExpiresAt() + this.stopped || !this.foreground + ? null + : (confirmation?.resumeExpiresAt ?? session.getResumeExpiresAt()) ) - this.scheduleDirectProbe() + this.directProbe.schedule() return { ok: true } } catch (error) { return { ok: false, error: session.getFailure() ?? toError(error) } } } - private scheduleDirectProbe(delayMs = DIRECT_PROBE_INTERVAL_MS): void { - if ( - this.stopped || - !this.foreground || - this.logical.getActivePath() !== 'relay' || - this.probeTimer - ) { - return - } - this.probeTimer = this.dependencies.setTimer(() => { - this.probeTimer = null - void this.probeDirect() - }, delayMs) - } - - private async probeDirect(): Promise { - if ( - this.stopped || - !this.foreground || - this.operationInFlight || - !this.hysteresis.canProbe(this.dependencies.now()) - ) { - this.scheduleDirectProbe() - return - } - this.operationInFlight = true - let successful: Awaited> = null - try { - const openDirect = this.dependencies.openDirect - successful = await openAuthenticatedDirectEndpoint(this.host, openDirect, 12_000) - if (!successful) { - this.hysteresis.recordDirectFailure(this.dependencies.now()) - return - } - if (!this.hysteresis.recordDirectSuccess(this.dependencies.now())) { - successful.client.close() - return - } - await this.logical.migrateTo(successful.client, successful.path) - successful = null - this.hysteresis.recordMigration(this.dependencies.now()) - this.leaseRotation.clear() - this.relayRotationPending = false - await this.rotateCredentialIfNeeded(this.relayReconnect.resetForDirectConnection()) - } finally { - successful?.client.close() - this.operationInFlight = false - // Why: a relay drop or backoff timer can arrive while the direct probe owns the mutex. - if (this.relayRotationPending || this.logical.getState() !== 'connected') { - void this.recoverRelay(this.relayRotationPending) - } - this.scheduleDirectProbe() - } - } - private async rotateCredentialIfNeeded(force = false): Promise { if ( this.stopped || @@ -321,11 +318,4 @@ export class MobileEndpointSupervisor { } } } - - private clearDirectProbeTimer(): void { - if (this.probeTimer) { - this.dependencies.clearTimer(this.probeTimer) - this.probeTimer = null - } - } } diff --git a/mobile/src/transport/mobile-relay-credential-selection.ts b/mobile/src/transport/mobile-relay-credential-selection.ts new file mode 100644 index 000000000..dcaca2080 --- /dev/null +++ b/mobile/src/transport/mobile-relay-credential-selection.ts @@ -0,0 +1,42 @@ +import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle' +import type { RelayReconnectController } from './mobile-relay-reconnect-controller' + +// Why: pairing recovery, re-pairing, or a raced rotation can land a fresh +// durable bundle after the supervisor snapshotted its copy; a gated retry must +// see it instead of dialing (or refusing to dial) on stale state forever. +// +// Adoption is decided by OUTCOME, not by version: renewals extend expiresAt +// without bumping current.version, and a re-pair restarts the version counter, +// so version comparison cannot tell fresh from stale. The durable bundle is +// adopted exactly when it yields a dialable (unexpired, non-rejected) +// credential while the in-memory one does not — which also means a revoked +// version can never be resurrected from a stale disk copy. +export async function selectDialableRelayCredentials(args: { + bundle: MobileRelayCredentialBundle | null + controller: RelayReconnectController + readBundle: () => Promise + onAdoptedFresherBundle: () => void +}): Promise<{ + bundle: MobileRelayCredentialBundle | null + credentials: MobileRelayCredentialBundle['current'][] +}> { + const memory = args.bundle + const memoryCredentials = memory + ? args.controller.eligibleCredentials(memory.current, memory.grace) + : [] + if (memoryCredentials.length > 0) { + // Why: in-memory can be newer than disk (a resume confirmation whose + // durable write failed); never let a stale disk copy shadow it. + return { bundle: memory, credentials: memoryCredentials } + } + const disk = await args.readBundle().catch(() => null) + if (disk) { + const diskCredentials = args.controller.eligibleCredentials(disk.current, disk.grace) + if (diskCredentials.length > 0) { + args.controller.acceptFreshCredential(disk.current.version) + args.onAdoptedFresherBundle() + return { bundle: disk, credentials: diskCredentials } + } + } + return { bundle: memory ?? disk, credentials: [] } +} diff --git a/mobile/src/transport/mobile-relay-lease-rotation-timer.ts b/mobile/src/transport/mobile-relay-lease-rotation-timer.ts index bf2a3dd4d..19aedfe8b 100644 --- a/mobile/src/transport/mobile-relay-lease-rotation-timer.ts +++ b/mobile/src/transport/mobile-relay-lease-rotation-timer.ts @@ -2,6 +2,13 @@ // little before the deadline (and retry shortly if a forced rotation didn't land) // so the session never lapses. Owns the single lease/rotation timer slot. const LEASE_ROTATION_MARGIN_MS = 30_000 +// Why: both ends must be clamped. The floor bounds any bad deadline to one +// forced rotation per minute instead of a sub-second loop; the ceiling keeps +// the delay far below setTimeout's 32-bit limit (a 30-day resume TTL minus the +// margin overflows int32 and fires at 1ms), at the cost of a harmless +// re-resume every few hours on long-lived sessions. +const LEASE_ROTATION_MIN_DELAY_MS = 60_000 +const LEASE_ROTATION_MAX_DELAY_MS = 6 * 60 * 60 * 1000 export type RelayLeaseRotationDependencies = { now: () => number @@ -23,9 +30,12 @@ export class RelayLeaseRotationTimer { if (!leaseExpiresAt) { return } - const delay = Math.max( - 1000, - leaseExpiresAt - this.dependencies.now() - LEASE_ROTATION_MARGIN_MS + const delay = Math.min( + LEASE_ROTATION_MAX_DELAY_MS, + Math.max( + LEASE_ROTATION_MIN_DELAY_MS, + leaseExpiresAt - this.dependencies.now() - LEASE_ROTATION_MARGIN_MS + ) ) this.arm(delay) } diff --git a/mobile/src/transport/mobile-relay-reconnect-controller.test.ts b/mobile/src/transport/mobile-relay-reconnect-controller.test.ts index d630f0db6..cc640d4d7 100644 --- a/mobile/src/transport/mobile-relay-reconnect-controller.test.ts +++ b/mobile/src/transport/mobile-relay-reconnect-controller.test.ts @@ -6,7 +6,9 @@ import { RelayReconnectController } from './mobile-relay-reconnect-controller' import type { StableLogicalRpcClient } from './stable-logical-rpc-client' vi.mock('react-native', () => ({ Platform: { OS: 'ios' } })) -vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) })) +vi.mock('expo-crypto', () => ({ + getRandomBytes: (length: number) => new Uint8Array(length) +})) describe('relay reconnect controller', () => { beforeEach(() => { @@ -47,16 +49,104 @@ describe('relay reconnect controller', () => { expect(onRetry).not.toHaveBeenCalled() }) - it('waits for an external signal after rejected E2EE authentication', () => { + it('reprobes slowly after rejected E2EE authentication instead of parking forever', () => { + // Why: on a relay-only phone a permanent gate is a permanent outage — the + // desktop can commit pairing credentials moments after the first rejection. const onRetry = vi.fn() const reconnect = createController(onRetry) reconnect.registerFailure(new MobileE2EEAuthenticationError()) expect(reconnect.shouldDefer()).toBe(true) - expect(vi.getTimerCount()).toBe(0) - vi.advanceTimersByTime(60_000) expect(onRetry).not.toHaveBeenCalled() + vi.advanceTimersByTime(59_000) + expect(onRetry).not.toHaveBeenCalled() + vi.advanceTimersByTime(1_000) + expect(onRetry).toHaveBeenCalledTimes(1) + // The reprobe tick passes the gate exactly once, then defers again. + expect(reconnect.shouldDefer()).toBe(false) + expect(reconnect.shouldDefer()).toBe(true) + }) + + it('keeps the fresh-credential gate reprobing after each failed gated attempt', () => { + const onRetry = vi.fn() + const reconnect = createController(onRetry) + + reconnect.registerFailure(new RelayOuterError(4401)) + expect(reconnect.shouldDefer()).toBe(true) + + vi.advanceTimersByTime(60_000) + expect(onRetry).toHaveBeenCalledTimes(1) + expect(reconnect.shouldDefer()).toBe(false) + // The gated attempt fails again with only rejected credentials on hand. + reconnect.registerFailure(new RelayOuterError(4401)) + + // The gated cadence escalates: the second reprobe waits twice as long. + vi.advanceTimersByTime(60_000) + expect(onRetry).toHaveBeenCalledTimes(1) + vi.advanceTimersByTime(60_000) + expect(onRetry).toHaveBeenCalledTimes(2) + }) + + it('resets the gated cadence when the app returns to the foreground', () => { + // Why: reopening the app is the strongest "conditions changed" signal a + // phone produces — it must not wait out an escalated 15-minute tick even + // when it cannot lift the credential gate itself. + const onRetry = vi.fn() + const reconnect = createController(onRetry) + const logical = { getState: () => 'disconnected' } as never + + reconnect.registerFailure(new RelayOuterError(4401)) + vi.advanceTimersByTime(60_000) + expect(onRetry).toHaveBeenCalledTimes(1) + // A failed gated attempt escalates the next tick beyond the base cadence. + reconnect.registerFailure(new RelayOuterError(4401)) + + reconnect.clear() + reconnect.handleForeground(logical, false) + expect(onRetry).toHaveBeenCalledTimes(2) + + expect(reconnect.shouldDefer()).toBe(true) + vi.advanceTimersByTime(60_000) + expect(onRetry).toHaveBeenCalledTimes(3) + }) + + it('does not let an orphaned reprobe timer swallow the next fast backoff', () => { + const onRetry = vi.fn() + const reconnect = createController(onRetry) + const logical = { getState: () => 'disconnected' } as never + + reconnect.registerFailure(new MobileE2EEAuthenticationError()) + // A foreground revival nudge clears the external-signal gate and its timer. + reconnect.handleForeground(logical, true) + expect(onRetry).toHaveBeenCalledTimes(1) + + // A plain transport failure must schedule its own fast retry, not wait + // out a leftover 60s reprobe timer. + reconnect.registerFailure(new RelayOuterError(4429)) + vi.advanceTimersByTime(600) + expect(onRetry).toHaveBeenCalledTimes(2) + }) + + it('lifts the fresh-credential gate when a durable bundle carries a new version', () => { + const onRetry = vi.fn() + const reconnect = createController(onRetry) + + reconnect.recordRejectedCredential(2) + reconnect.armCredentialReprobe() + expect(reconnect.shouldDefer()).toBe(true) + + reconnect.acceptFreshCredential(2) + expect(reconnect.shouldDefer()).toBe(true) + + reconnect.acceptFreshCredential(3) + expect(reconnect.shouldDefer()).toBe(false) + expect( + reconnect.eligibleCredentials( + { token: 'fresh', version: 3, expiresAt: Number.MAX_SAFE_INTEGER }, + null + ) + ).toHaveLength(1) }) it('upgrades host-revival gating to fresh credentials without later downgrading it', () => { diff --git a/mobile/src/transport/mobile-relay-reconnect-controller.ts b/mobile/src/transport/mobile-relay-reconnect-controller.ts index 6f586c174..f2a7cad6b 100644 --- a/mobile/src/transport/mobile-relay-reconnect-controller.ts +++ b/mobile/src/transport/mobile-relay-reconnect-controller.ts @@ -17,6 +17,12 @@ const RELAY_BACKOFF_CEILING_MS = 30_000 const RELAY_STABLE_CONNECTION_MS = RELAY_BACKOFF_CEILING_MS const RELAY_HOST_OFFLINE_RETRY_MIN_MS = 5_000 const RELAY_HOST_OFFLINE_RETRY_MAX_MS = 15_000 +// Why: gates must slow recovery down, never end it — a relay-only phone has no +// direct path to refresh credentials, so a timerless gate is a permanent outage. +// The cadence escalates and jitters so a permanently revoked device is not a +// forever one-minute beacon and a fleet-wide gating event cannot phase-align. +const RELAY_GATE_REPROBE_BASE_MS = 60_000 +const RELAY_GATE_REPROBE_CEILING_MS = 15 * 60_000 export type RelayReconnectDependencies = { now: () => number @@ -34,6 +40,8 @@ export class RelayReconnectController { private timer: ReturnType | null = null private activeSession: MobileRelayRpcSession | null = null private recoveryGate: RecoveryGate | null = null + private gateReprobePending = false + private gateReprobeStreak = 0 private readonly rejectedCredentialVersions = new Set() constructor( @@ -43,12 +51,15 @@ export class RelayReconnectController { handleForeground(logical: StableLogicalRpcClient, wasForeground: boolean): void { if (!wasForeground) { - // Why: an app resume is a fresh signal, unlike repeated network-flap nudges. + // Why: an app resume is a fresh signal, unlike repeated network-flap + // nudges — it resets the gated cadence even when it cannot lift the + // credential gate, so reopening the app never waits out a 15min tick. + this.gateReprobeStreak = 0 if (this.recoveryGate !== 'fresh-credential') { this.reset() } } else if (this.recoveryGate === 'external-signal') { - this.recoveryGate = null + this.liftGate() } if ( wasForeground && @@ -87,8 +98,7 @@ export class RelayReconnectController { this.activeSession = session this.activeRelayConnectedAt = this.dependencies.now() this.nextAttemptAt = 0 - this.recoveryGate = null - this.clearTimer() + this.liftGate() } resetForDirectConnection(): boolean { @@ -97,10 +107,14 @@ export class RelayReconnectController { this.activeSession = null this.activeRelayConnectedAt = null if (needsCredentialRefresh) { - // Why: the rejected credential stays unusable until its replacement is durable. + // Why: the rejected credential stays unusable until its replacement is + // durable. No reprobe timer here — direct is live, rotation over it + // clears the gate, and any later state failure re-arms via shouldDefer. this.consecutiveFailures = 0 this.nextAttemptAt = 0 this.recoveryGate = 'fresh-credential' + this.gateReprobePending = false + this.gateReprobeStreak = 0 this.clearTimer() } else { this.reset() @@ -128,10 +142,36 @@ export class RelayReconnectController { if (eligible.length === 0 && this.rejectedCredentialVersions.size > 0) { this.recoveryGate = 'fresh-credential' this.clearTimer() + this.scheduleGateReprobe() } return eligible } + // For callers that found no dialable credential at all: keep a slow retry + // alive so a later durable write can recover. + armCredentialReprobe(): void { + if (this.rejectedCredentialVersions.size > 0) { + this.recoveryGate = 'fresh-credential' + this.clearTimer() + this.scheduleGateReprobe() + return + } + // Why: a merely missing or expired bundle must not enter the credential + // gate — that gate forces a rotation on the next direct connect. A plain + // cooldown retries the read on the same escalating cadence. + const delay = this.gateReprobeDelayMs() + this.nextAttemptAt = this.dependencies.now() + delay + this.clearTimer() + this.scheduleReprobeTick(delay, false) + } + + // A durable bundle whose current version is not rejected reopens the gate. + acceptFreshCredential(version: number): void { + if (this.recoveryGate === 'fresh-credential' && !this.rejectedCredentialVersions.has(version)) { + this.liftGate() + } + } + recordRejectedCredential(version: number): void { this.rejectedCredentialVersions.add(version) } @@ -154,6 +194,12 @@ export class RelayReconnectController { // re-dial. Arms the self-scheduled retry so recovery still happens on its own. shouldDefer(): boolean { if (this.recoveryGate) { + if (this.gateReprobePending) { + // Why: the slow reprobe tick gets exactly one attempt through the gate. + this.gateReprobePending = false + return false + } + this.scheduleGateReprobe() return true } if (this.dependencies.now() < this.nextAttemptAt) { @@ -173,8 +219,9 @@ export class RelayReconnectController { this.recoveryGate === 'fresh-credential' || (this.recoveryGate === 'external-signal' && recovery?.kind !== 'disable-relay-credential') ) { - // Why: only the gate's external signal can make a known-fatal recovery retryable. + // Why: a failed reprobe stays gated, but the slow cadence must keep going. this.clearTimer() + this.scheduleGateReprobe() return } const now = this.dependencies.now() @@ -192,15 +239,20 @@ export class RelayReconnectController { recovery?.kind === 'retry-after-host-offline' ? this.hostOfflineDelayMs() : this.delayMs() this.nextAttemptAt = now + delay if (error instanceof MobileE2EEAuthenticationError) { - // Why: pairing state cannot change on a timer; polling only wakes the radio. + // Why: an E2EE rejection is usually pairing revocation, but it also fires + // transiently right after pairing while the desktop commits credentials — + // reprobe slowly instead of waiting forever for a UI nudge. this.recoveryGate = 'external-signal' this.clearTimer() + this.scheduleGateReprobe() return } if (recovery?.kind === 'disable-relay-credential') { - // Why: a rejected outer credential cannot recover until direct connectivity refreshes it. + // Why: never redial a rejected credential fast, but keep a slow reprobe + // alive — the caller re-reads durable state before each gated attempt. this.recoveryGate = 'fresh-credential' this.clearTimer() + this.scheduleGateReprobe() return } if (recovery?.kind === 'retry-after-host-offline') { @@ -235,12 +287,12 @@ export class RelayReconnectController { this.consecutiveFailures = 0 this.activeRelayConnectedAt = null this.nextAttemptAt = 0 - this.recoveryGate = null - this.clearTimer() + this.liftGate() } clear(): void { this.clearTimer() + this.gateReprobePending = false this.activeSession = null this.activeRelayConnectedAt = null } @@ -263,6 +315,45 @@ export class RelayReconnectController { }, delay) } + private scheduleGateReprobe(): void { + this.scheduleReprobeTick(this.gateReprobeDelayMs(), true) + } + + private scheduleReprobeTick(delay: number, mintToken: boolean): void { + if (this.timer) { + return + } + this.timer = this.dependencies.setTimer(() => { + this.timer = null + // Why: the streak advances once per fired tick — arm attempts within one + // cycle recompute the same delay instead of triple-escalating it. + this.gateReprobeStreak = Math.min(this.gateReprobeStreak + 1, 8) + // Why: the tick token is only minted while its gate still holds; a later + // gate must not spend a stale token and bypass its own cooldown. + if (mintToken && this.recoveryGate) { + this.gateReprobePending = true + } + this.onRetry() + }, delay) + } + + private gateReprobeDelayMs(): number { + const base = Math.min( + RELAY_GATE_REPROBE_CEILING_MS, + RELAY_GATE_REPROBE_BASE_MS * 2 ** Math.min(this.gateReprobeStreak, 8) + ) + return Math.floor(base * (0.75 + 0.5 * this.jitterFraction())) + } + + // Why: clearing a gate must also drop its timer, pending tick, and cadence — + // an orphaned reprobe timer would otherwise swallow the next fast backoff. + private liftGate(): void { + this.recoveryGate = null + this.gateReprobePending = false + this.gateReprobeStreak = 0 + this.clearTimer() + } + private delayMs(): number { const exponent = Math.max(0, this.consecutiveFailures - 1) const cap = Math.min(RELAY_BACKOFF_CEILING_MS, RELAY_BACKOFF_BASE_MS * 2 ** exponent) diff --git a/mobile/src/transport/mobile-relay-recovery-log.ts b/mobile/src/transport/mobile-relay-recovery-log.ts new file mode 100644 index 000000000..9b373cd11 --- /dev/null +++ b/mobile/src/transport/mobile-relay-recovery-log.ts @@ -0,0 +1,22 @@ +import type { ConnectionLogSink } from './types' + +export type RelayRecoveryLog = (message: string, detail?: string) => void + +// Why: relay recovery failed silently in production for weeks; every decision +// must reach logcat and the in-app connection log. +export function createRelayRecoveryLog( + now: () => number, + onLog?: ConnectionLogSink +): RelayRecoveryLog { + let sequence = 0 + return (message, detail) => { + console.log(`[relay] ${message}`, detail ?? '') + onLog?.({ + id: `relay-${++sequence}`, + ts: now(), + level: 'info', + message: `Relay: ${message}`, + ...(detail ? { detail } : {}) + }) + } +} diff --git a/mobile/src/transport/mobile-relay-rpc-session.test.ts b/mobile/src/transport/mobile-relay-rpc-session.test.ts index 54ae83874..f5fa49a78 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.test.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.test.ts @@ -119,7 +119,7 @@ describe('mobile relay RPC session', () => { }) expect(confirmationRequest.params).not.toHaveProperty('relayDeviceId') expect(confirmationRequest.params).not.toHaveProperty('acceptedCredentialVersion') - expect(session.getLeaseExpiresAt()).toEqual(expect.any(Number)) + expect(session.getAttachDeadlineAt()).toEqual(expect.any(Number)) }) it('rejects a mismatched outer credential version and closes the physical link', () => { diff --git a/mobile/src/transport/mobile-relay-rpc-session.ts b/mobile/src/transport/mobile-relay-rpc-session.ts index a614e8333..ac0650122 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.ts @@ -19,7 +19,10 @@ type PendingRequest = { } export type MobileRelayRpcSession = RpcClient & { - getLeaseExpiresAt(): number | null + // The cell's attach-reservation deadline (~10s). Diagnostics only — never + // schedule anything from it; rotation keys off getResumeExpiresAt(). + getAttachDeadlineAt(): number | null + getResumeExpiresAt(): number | null getResumeConfirmation(): DeviceResumeConfirmed | null getFailure(): Error | null } @@ -40,7 +43,8 @@ export function connectMobileRelayRpcSession(args: { let state: ConnectionState = 'connecting' let requestCounter = 0 let lastConnectedAt: number | null = null - let leaseExpiresAt: number | null = null + let attachDeadlineAt: number | null = null + let resumeExpiresAt: number | null = null let resumeConfirmation: DeviceResumeConfirmed | null = null let failure: Error | null = null let closed = false @@ -65,7 +69,8 @@ export function connectMobileRelayRpcSession(args: { fail(new Error('relay resume credential version mismatch')) return } - leaseExpiresAt = hello.leaseExpiresAt + attachDeadlineAt = hello.leaseExpiresAt + resumeExpiresAt = hello.resumeExpiresAt publishState('handshaking') }, onAuthenticated: () => void confirmResume(), @@ -109,7 +114,8 @@ export function connectMobileRelayRpcSession(args: { streams.clear() publishState('disconnected') }, - getLeaseExpiresAt: () => leaseExpiresAt, + getAttachDeadlineAt: () => attachDeadlineAt, + getResumeExpiresAt: () => resumeExpiresAt, getResumeConfirmation: () => resumeConfirmation, getFailure: () => failure } @@ -131,6 +137,7 @@ export function connectMobileRelayRpcSession(args: { throw new Error('relay resume confirmation missing') } resumeConfirmation = result.resumeConfirmation + resumeExpiresAt = result.resumeConfirmation.resumeExpiresAt lastConnectedAt = Date.now() publishState('connected') } catch (error) { diff --git a/mobile/src/transport/mobile-relay-runtime-failover.test.ts b/mobile/src/transport/mobile-relay-runtime-failover.test.ts new file mode 100644 index 000000000..dcb8ba194 --- /dev/null +++ b/mobile/src/transport/mobile-relay-runtime-failover.test.ts @@ -0,0 +1,454 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { connect, type RpcClient } from './rpc-client' +import { + createStableLogicalRpcClient, + type MobileConnectionPath, + type StableLogicalRpcClient +} from './stable-logical-rpc-client' +import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel' +import { RelayOuterError } from './mobile-relay-e2ee-link' +import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle' +import type { MobileRelayRpcSession } from './mobile-relay-rpc-session' +import { + MobileEndpointSupervisor, + type MobileEndpointSupervisorDependencies +} from './mobile-endpoint-supervisor' +import type { ConnectionState, HostProfile, RpcResponse } from './types' + +// Regression suite for the 2026-08 field failure: a phone paired over the +// relay whose direct LAN endpoint is unreachable (Tailscale off) dialed the +// LAN endpoint forever and never recovered a relay runtime session. + +vi.mock('react-native', () => ({ Platform: { OS: 'android' } })) +vi.mock('expo-secure-store', () => ({ + WHEN_UNLOCKED_THIS_DEVICE_ONLY: 'when-unlocked' +})) +vi.mock('expo-crypto', () => ({ + getRandomBytes: (length: number) => new Uint8Array(length) +})) + +vi.mock('./e2ee', () => ({ + generateKeyPair: () => ({ + publicKey: new Uint8Array(32), + secretKey: new Uint8Array(32) + }), + deriveSharedKey: () => new Uint8Array(32), + publicKeyFromBase64: () => new Uint8Array(32), + publicKeyToBase64: () => 'client-public-key', + encrypt: (plaintext: string) => plaintext, + decrypt: (raw: string) => raw, + decryptBytes: (bytes: Uint8Array) => bytes +})) + +class FakeSession implements RpcClient { + readonly sendRequest = vi.fn( + async (): Promise => ({ + id: 'rpc-1', + ok: true, + result: {}, + _meta: { runtimeId: 'runtime-1' } + }) + ) + readonly subscribe = vi.fn(() => () => {}) + readonly updateTerminalSubscriptionViewport = vi.fn() + readonly notifyForeground = vi.fn() + readonly close = vi.fn() + private readonly listeners = new Set<(state: ConnectionState) => void>() + + constructor(private state: ConnectionState) {} + + getState = () => this.state + getReconnectAttempt = () => 0 + getLastConnectedAt = () => null + onStateChange = (listener: (state: ConnectionState) => void) => { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + + publishState(state: ConnectionState): void { + this.state = state + for (const listener of this.listeners) { + listener(state) + } + } +} + +class FakeRelaySession extends FakeSession implements MobileRelayRpcSession { + constructor( + state: ConnectionState, + private readonly failure: Error | null = null + ) { + super(state) + } + // Why: production-realistic constants — fictional fake values hid three + // live defects in this subsystem (latch, churn, int32 timer overflow). + getAttachDeadlineAt = () => Date.now() + 10_000 + getResumeExpiresAt = () => Date.now() + 30 * 24 * 3_600_000 + getResumeConfirmation = () => null + getFailure = () => this.failure +} + +class FakeLogicalClient extends FakeSession implements StableLogicalRpcClient { + private path: MobileConnectionPath + + constructor(state: ConnectionState, path: MobileConnectionPath) { + super(state) + this.path = path + } + + migrateTo = vi.fn(async (session: RpcClient, path: MobileConnectionPath) => { + if (session.getState() !== 'connected') { + session.close() + throw new Error(`replacement session ${session.getState()}`) + } + this.path = path + this.publishState('connected') + }) + suspendActiveSession = vi.fn(() => this.publishState('disconnected')) + getActivePath = () => this.path +} + +const relay = { + v: 1 as const, + directorUrl: 'https://relay.onorca.dev', + cellUrl: 'https://relay-c1.onorca.dev', + assignmentEpoch: 7, + relayHostId: 'AbCdEf0123_-xyZ9', + e2eeFraming: 2 as const +} + +const DIRECT_ENDPOINT = 'ws://100.88.90.25:6768' + +const host: HostProfile = { + id: 'host-1', + name: 'Blue Whale', + endpoint: DIRECT_ENDPOINT, + deviceToken: 'device-token', + publicKeyB64: 'A'.repeat(44), + lastConnected: 1, + endpoints: [ + { id: 'direct-primary', kind: 'lan', url: DIRECT_ENDPOINT }, + { + id: 'relay-primary', + kind: 'relay', + url: 'wss://relay-c1.onorca.dev/v1/connect/id' + } + ], + relayHostId: relay.relayHostId, + relay +} + +function bundleWith(version: number, expiresAt: number): MobileRelayCredentialBundle { + return { + v: 1, + hostId: host.id, + deviceToken: host.deviceToken, + current: { + token: `token-v${version}`.padEnd(43, 'A'), + hash: 'B'.repeat(43), + version, + expiresAt + } + } +} + +function dependencies( + overrides: Partial = {} +): MobileEndpointSupervisorDependencies { + return { + openDirect: vi.fn(() => new FakeSession('connected')), + openRelay: vi.fn(() => new FakeRelaySession('connected')), + resolveRelay: vi.fn(async ({ relay }) => relay), + readBundle: vi.fn(async () => bundleWith(2, Number.MAX_SAFE_INTEGER)), + writeBundle: vi.fn(async () => {}), + saveHost: vi.fn(async () => {}), + now: Date.now, + randomBytes: (length: number) => new Uint8Array(length), + setTimer: setTimeout, + clearTimer: clearTimeout, + ...overrides + } +} + +describe('relay runtime recovery without direct connectivity', () => { + beforeEach(() => { + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-08-02T12:00:00Z')) + }) + + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it('recovers from a rejected outer credential once a fresher bundle is durable', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + const openRelay = vi + .fn() + .mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4401))) + .mockImplementation(() => new FakeRelaySession('connected')) + const readBundle = vi + .fn(async () => bundleWith(3, Number.MAX_SAFE_INTEGER)) + .mockResolvedValueOnce(bundleWith(2, Number.MAX_SAFE_INTEGER)) + const deps = dependencies({ openRelay, readBundle }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(openRelay).toHaveBeenCalledOnce() + + // Pre-fix, the fresh-credential gate latched here with no timer and no exit. + await vi.advanceTimersByTimeAsync(60_000) + expect(openRelay).toHaveBeenCalledTimes(2) + expect(openRelay).toHaveBeenLastCalledWith( + relay, + expect.objectContaining({ version: 3 }), + expect.any(String) + ) + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('recovers when the credential bundle was unreadable at supervisor start', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + const readBundle = vi + .fn(async () => bundleWith(2, Number.MAX_SAFE_INTEGER)) + .mockResolvedValueOnce(null) + const deps = dependencies({ readBundle }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + // Pre-fix, a null first read killed relay recovery for the process lifetime. + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + + expect(deps.openRelay).toHaveBeenCalled() + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('recovers from an expired-at-start bundle after a fresh credential lands', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + const expired = bundleWith(2, Date.now() - 1) + const readBundle = vi.fn(async () => expired) + const deps = dependencies({ readBundle }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(deps.openRelay).not.toHaveBeenCalled() + + readBundle.mockImplementation(async () => bundleWith(3, Number.MAX_SAFE_INTEGER)) + // Pre-fix, an expired bundle produced a silent no-op with nothing scheduled. + await vi.advanceTimersByTimeAsync(60_000) + + expect(deps.openRelay).toHaveBeenCalledOnce() + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('retries after an E2EE authentication rejection without a UI nudge', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + const openRelay = vi + .fn() + .mockReturnValueOnce( + new FakeRelaySession('disconnected', new MobileE2EEAuthenticationError()) + ) + .mockImplementation(() => new FakeRelaySession('connected')) + const deps = dependencies({ openRelay }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(openRelay).toHaveBeenCalledOnce() + + // Pre-fix, this state waited indefinitely for a foreground/navigation event. + await vi.advanceTimersByTimeAsync(60_000) + expect(openRelay).toHaveBeenCalledTimes(2) + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('adopts a durable renewal that extends expiry without bumping the version', async () => { + // Why: resume confirmations and pairing recovery renew expiresAt while + // keeping current.version — version comparison cannot see this freshness. + const logical = new FakeLogicalClient('disconnected', 'lan') + const expired = bundleWith(2, Date.now() - 1) + const readBundle = vi + .fn(async () => bundleWith(2, Number.MAX_SAFE_INTEGER)) + .mockResolvedValueOnce(expired) + .mockResolvedValueOnce(expired) + const deps = dependencies({ readBundle }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(deps.openRelay).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(60_000) + expect(deps.openRelay).toHaveBeenCalledOnce() + expect(deps.openRelay).toHaveBeenLastCalledWith( + relay, + expect.objectContaining({ version: 2 }), + expect.any(String) + ) + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('adopts a re-paired credential whose version counter restarted', async () => { + // Why: re-pairing overwrites the same keychain slot with a NEW credential + // record whose counter restarts at 1 — lower than the rejected version. + const logical = new FakeLogicalClient('disconnected', 'lan') + const openRelay = vi + .fn() + .mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4401))) + .mockImplementation(() => new FakeRelaySession('connected')) + const readBundle = vi + .fn(async () => bundleWith(1, Number.MAX_SAFE_INTEGER)) + .mockResolvedValueOnce(bundleWith(4, Number.MAX_SAFE_INTEGER)) + const deps = dependencies({ openRelay, readBundle }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(openRelay).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(60_000) + expect(openRelay).toHaveBeenCalledTimes(2) + expect(openRelay).toHaveBeenLastCalledWith( + relay, + expect.objectContaining({ version: 1 }), + expect.any(String) + ) + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('does not churn relay sessions off the cell attach-reservation deadline', async () => { + // Why: the relay-hello's leaseExpiresAt is a ~10s attach deadline. Keying + // rotation off it replaced the session every second, killing any RPC + // slower than the cycle (the field symptom: "Worktree list unavailable"). + const logical = new FakeLogicalClient('disconnected', 'lan') + const openRelay = vi.fn(() => new FakeRelaySession('connected')) + // Direct stays unreachable, as in the field — return probes must not + // confuse the churn measurement by migrating back to direct. + const openDirect = vi.fn(() => new FakeSession('disconnected')) + const deps = dependencies({ openRelay, openDirect }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(openRelay).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(5 * 60_000) + expect(openRelay).toHaveBeenCalledOnce() + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) + + it('recovers immediately on a background/foreground cycle after an E2EE rejection', async () => { + const logical = new FakeLogicalClient('disconnected', 'lan') + const openRelay = vi + .fn() + .mockReturnValueOnce( + new FakeRelaySession('disconnected', new MobileE2EEAuthenticationError()) + ) + .mockImplementation(() => new FakeRelaySession('connected')) + const deps = dependencies({ openRelay }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(0) + expect(openRelay).toHaveBeenCalledOnce() + + supervisor.setForeground(false) + supervisor.setForeground(true) + await vi.advanceTimersByTimeAsync(0) + + expect(openRelay).toHaveBeenCalledTimes(2) + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + }) +}) + +// The field failure's first symptom: a real direct rpc-client dialing an +// unroutable LAN endpoint (instant 1006) while relay credentials sit unused. +class DeadSocket { + static constructed: string[] = [] + onopen: (() => void) | null = null + onclose: ((event: { code?: number; reason?: string }) => void) | null = null + onerror: ((event: unknown) => void) | null = null + onmessage: ((event: { data: unknown }) => void) | null = null + readyState = 0 + + constructor(readonly url: string) { + DeadSocket.constructed.push(url) + setTimeout(() => { + this.readyState = 3 + this.onclose?.({ code: 1006, reason: '' }) + }, 50) + } + + send(): void {} + close(): void { + this.readyState = 3 + } +} + +describe('failover with a real direct rpc-client', () => { + beforeEach(() => { + DeadSocket.constructed = [] + vi.stubGlobal('WebSocket', DeadSocket) + vi.spyOn(console, 'log').mockImplementation(() => {}) + vi.useFakeTimers() + vi.setSystemTime(new Date('2026-08-02T12:00:00Z')) + }) + + afterEach(() => { + vi.useRealTimers() + vi.unstubAllGlobals() + vi.restoreAllMocks() + }) + + it('migrates the logical client to relay after the direct dial fails', async () => { + const logical = createStableLogicalRpcClient( + connect(DIRECT_ENDPOINT, host.deviceToken, host.publicKeyB64), + 'tailscale' as MobileConnectionPath + ) + const deps = dependencies() + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + await supervisor.start() + await vi.advanceTimersByTimeAsync(3_000) + + expect(deps.openRelay).toHaveBeenCalled() + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + logical.close() + }) + + it('reaches relay even when the bundle read settles after the first direct failure', async () => { + const logical = createStableLogicalRpcClient( + connect(DIRECT_ENDPOINT, host.deviceToken, host.publicKeyB64), + 'tailscale' as MobileConnectionPath + ) + const deps = dependencies({ + readBundle: vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 200)) + return bundleWith(2, Number.MAX_SAFE_INTEGER) + }) + }) + const supervisor = new MobileEndpointSupervisor(logical, host, deps) + + const started = supervisor.start() + await vi.advanceTimersByTimeAsync(300) + await started + await vi.advanceTimersByTimeAsync(3_000) + + expect(deps.openRelay).toHaveBeenCalled() + expect(logical.getActivePath()).toBe('relay') + supervisor.stop() + logical.close() + }) +})