xiaoxue_admin
/
orca
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(mobile): recover unreliable relay connections (#10709) 

* fix(mobile): recover unreliable relay connections

* test(mobile): use valid raster preview fixtures

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
Jinwoo Hong 2026-07-26 00:48:06 -07:00 committed by GitHub
parent 4ff2f51782
commit 07671d4a06
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 305 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
config/reliability-gates.jsonc
View File

@ -1,6 +1,6 @@
{
"schemaVersion": 1,
"updatedAt": "2026-07-22",
"updatedAt": "2026-07-25",
"policy": {
"maturityLevels": [
"experimental",
@ -16,6 +16,131 @@
}
},
"gates": [
{
"id": "mobile-relay.endpoint-recovery",
"title": "Mobile relay recovery retries offline hosts and races direct endpoints",
"maturity": "experimental",
"protection": "partial",
"owner": "mobile-runtime",
"layer": "shared-mobile-transport-contract",
"surfaces": [
"paired mobile reconnect",
"cloud relay host-offline recovery",
"LAN direct endpoint",
"Tailscale direct endpoint"
],
"platforms": [
"ios",
"android",
"macos",
"linux",
"windows"
],
"providers": [
"lan",
"tailscale",
"cloud-relay"
],
"coveredPlatforms": [
"macos"
],
"coveredProviders": [
"lan",
"tailscale",
"cloud-relay"
],
"coverageNotes": "Deterministic TypeScript tests cover shared close-code policy, foreground retry timers, direct-winner cancellation, and concurrent LAN/Tailscale authentication. Physical iOS/Android radios, GFE, and production relay recovery remain live-test gaps.",
"motivatingLinks": [
"https://github.com/stablyai/orca-cloud/pull/96"
],
"invariant": "A foregrounded paired phone must recover from relay HOST_OFFLINE without a foreground or network-change signal, while direct recovery must select the first authenticated configured LAN or Tailscale endpoint without serial timeout delays. Backgrounding, direct success, or stop must cancel pending work, and losing probes must close without affecting the winner.",
"oracle": "Inject deterministic relay close codes, random bytes, fake timers, and independently controlled direct clients. Require HOST_OFFLINE to replace any faster transport timer with one 5-15 second retry, require no retry before the selected delay, race all unique non-relay endpoints, select the first authenticated path, close every loser exactly once, and retain no retry after direct connectivity wins.",
"commands": [
"pnpm --dir mobile exec vitest run --root .. mobile/src/transport/mobile-direct-endpoint-probe.test.ts mobile/src/transport/mobile-relay-reconnect-controller.test.ts mobile/src/transport/mobile-endpoint-supervisor.test.ts",
"pnpm exec vitest run --config config/vitest.config.ts src/shared/mobile-relay-close-codes.test.ts --reporter=dot"
],
"testFiles": [
"mobile/src/transport/mobile-direct-endpoint-probe.test.ts",
"mobile/src/transport/mobile-relay-reconnect-controller.test.ts",
"mobile/src/transport/mobile-endpoint-supervisor.test.ts",
"src/shared/mobile-relay-close-codes.test.ts"
],
"assertionRefs": [
{
"file": "mobile/src/transport/mobile-direct-endpoint-probe.test.ts",
"assertions": [
"a reachable Tailscale endpoint authenticates without waiting for a stale primary LAN timeout",
"the stale direct candidate closes while the authenticated winner stays open"
]
},
{
"file": "mobile/src/transport/mobile-relay-reconnect-controller.test.ts",
"assertions": [
"HOST_OFFLINE replaces a pending capacity retry with the bounded host-offline delay",
"direct connectivity cancels the pending relay retry"
]
},
{
"file": "mobile/src/transport/mobile-endpoint-supervisor.test.ts",
"assertions": [
"a foregrounded supervisor retries HOST_OFFLINE without an external lifecycle signal"
]
},
{
"file": "src/shared/mobile-relay-close-codes.test.ts",
"assertions": [
"HOST_OFFLINE maps to self-healing full-jitter recovery"
]
}
],
"evidenceRuns": [
{
"date": "2026-07-25",
"runner": "local",
"platform": "macos",
"command": "pnpm --dir mobile exec vitest run --root .. mobile/src/transport/mobile-direct-endpoint-probe.test.ts mobile/src/transport/mobile-relay-reconnect-controller.test.ts mobile/src/transport/mobile-endpoint-supervisor.test.ts",
"result": "passed",
"durationSeconds": 0.89,
"summary": "Three focused mobile transport files passed with 39 assertions."
},
{
"date": "2026-07-25",
"runner": "local",
"platform": "macos",
"command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/mobile-relay-close-codes.test.ts --reporter=dot",
"result": "passed",
"durationSeconds": 0.19,
"summary": "The shared close-code contract passed with five assertions."
}
],
"runtimeBudget": {
"p95Seconds": 5,
"scope": "focused shared and mobile transport unit tests"
},
"flakeHistory": {
"status": "unknown",
"evidence": "Two deterministic local runs exist; CI and soak history are not yet available."
},
"redGreenEvidence": {
"status": "complete",
"evidence": "The prior external-signal HOST_OFFLINE policy fails the retry oracle, and the prior serial direct probe fails the first-authenticated-endpoint timing oracle. Both pass with the candidate behavior."
},
"performanceBudget": {
"required": true,
"evidence": "All configured direct candidates start in one turn, the first authenticated candidate wins after 100 ms in the deterministic test, and every losing client is closed. A physical-device radio and battery budget is still required before promotion."
},
"promotionCriteria": [
"Collect 100 consecutive CI passes or 14 days of soak history.",
"Run paired iOS and Android recovery through production-like GFE HOST_OFFLINE responses.",
"Measure reconnect radio and battery impact for the 5-15 second foreground retry window."
],
"knownGaps": [
"No physical iOS or Android device was exercised.",
"The deterministic transport seam does not measure production GFE, carrier NAT, DNS, TLS, or Cloud SQL latency.",
"Background-to-foreground recovery remains covered by existing supervisor tests but lacks a physical sleep/wake run."
],
"demotionRule": "Keep experimental or demote if focused tests flake, HOST_OFFLINE can park indefinitely, direct probes serialize configured endpoints, loser cleanup leaks clients, or physical-device radio cost exceeds the measured budget."
},
{
"id": "git-worktree.refresh-event-semantics",
"title": "Index-only Git metadata cannot trigger structural worktree refresh fanout",

19
mobile/src/files/mobile-diff-image-preview.test.ts
View File

@ -1,17 +1,20 @@
import { describe, expect, it } from 'vitest'
import { mobileDiffImageDataUri } from './mobile-diff-image-preview'
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/l8sm7wAAAABJRU5ErkJggg=='
describe('mobileDiffImageDataUri', () => {
it('renders a modified image diff from the post-change bytes', () => {
expect(
mobileDiffImageDataUri({
kind: 'binary',
originalContent: 'b2xk',
modifiedContent: 'bmV3',
originalContent: PNG_BASE64,
modifiedContent: PNG_BASE64,
isImage: true,
mimeType: 'image/png'
})
).toBe('data:image/png;base64,bmV3')
).toBe(`data:image/png;base64,${PNG_BASE64}`)
})
it('renders an added image diff (no original) from the modified bytes', () => {
@ -19,26 +22,26 @@ describe('mobileDiffImageDataUri', () => {
mobileDiffImageDataUri({
kind: 'binary',
originalContent: '',
modifiedContent: 'bmV3',
modifiedContent: PNG_BASE64,
isImage: true,
mimeType: 'image/png'
})
).toBe('data:image/png;base64,bmV3')
).toBe(`data:image/png;base64,${PNG_BASE64}`)
})
it('falls back to the original bytes for a proven deletion (modifiedDeleted)', () => {
expect(
mobileDiffImageDataUri({
kind: 'binary',
originalContent: 'b2xk',
originalContent: PNG_BASE64,
originalIsBinary: true,
modifiedContent: '',
modifiedIsBinary: false,
modifiedDeleted: true,
isImage: true,
mimeType: 'image/jpeg'
mimeType: 'image/png'
})
).toBe('data:image/jpeg;base64,b2xk')
).toBe(`data:image/png;base64,${PNG_BASE64}`)
})
// The reviewer's read-failure case: a relay/SSH read returns an empty modified

21
mobile/src/files/mobile-file-tab-doc.test.ts
View File

@ -2,6 +2,9 @@ import { describe, expect, it } from 'vitest'
import type { RpcResponse } from '../transport/types'
import { resolveMobileFileTabDoc } from './mobile-file-tab-doc'
const PNG_BASE64 =
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/l8sm7wAAAABJRU5ErkJggg=='
function ok(result: unknown): RpcResponse {
return { id: 'x', ok: true, result, _meta: { runtimeId: 'r' } }
}
@ -49,8 +52,8 @@ describe('resolveMobileFileTabDoc', () => {
const client = clientOf({
'git.diff': ok({
kind: 'binary',
originalContent: 'b2xk',
modifiedContent: 'bmV3',
originalContent: PNG_BASE64,
modifiedContent: PNG_BASE64,
modifiedIsBinary: true,
isImage: true,
mimeType: 'image/png'
@ -61,7 +64,11 @@ describe('resolveMobileFileTabDoc', () => {
relativePath: 'm1.png',
diffSource: 'unstaged'
})
expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' })
expect(doc).toEqual({
status: 'ready',
kind: 'image',
dataUri: `data:image/png;base64,${PNG_BASE64}`
})
})
it('throws binary_file for an image modify whose bytes are empty (no stale fallback)', async () => {
@ -89,10 +96,14 @@ describe('resolveMobileFileTabDoc', () => {
it('renders a live image preview via files.readPreview', async () => {
const client = clientOf({
'files.readPreview': ok({ content: 'bmV3', isImage: true, mimeType: 'image/png' })
'files.readPreview': ok({ content: PNG_BASE64, isImage: true, mimeType: 'image/png' })
})
const doc = await resolveMobileFileTabDoc(client, { ...WT, relativePath: 'logo.png' })
expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' })
expect(doc).toEqual({
status: 'ready',
kind: 'image',
dataUri: `data:image/png;base64,${PNG_BASE64}`
})
expect(client.calls).toEqual(['files.readPreview'])
})

76
mobile/src/transport/mobile-direct-endpoint-probe.test.ts Normal file
View File

@ -0,0 +1,76 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { RpcClient } from './rpc-client'
import { openAuthenticatedDirectEndpoint } from './mobile-direct-endpoint-probe'
import type { ConnectionState, HostProfile, RpcResponse } from './types'
class FakeClient implements RpcClient {
readonly sendRequest = vi.fn(
async (): Promise<RpcResponse> => ({
id: 'rpc-1',
ok: true,
result: {},
_meta: { runtimeId: 'runtime-1' }
})
)
readonly subscribe = vi.fn(() => () => {})
readonly updateTerminalSubscriptionViewport = vi.fn()
readonly notifyForeground = vi.fn()
readonly close = vi.fn(() => this.publishState('disconnected'))
private readonly listeners = new Set<(state: ConnectionState) => void>()
constructor(private state: ConnectionState) {}
getState = () => this.state
getReconnectAttempt = () => 0
getLastConnectedAt = () => null
onStateChange = (listener: (state: ConnectionState) => void) => {
this.listeners.add(listener)
return () => this.listeners.delete(listener)
}
publishState(state: ConnectionState): void {
this.state = state
for (const listener of this.listeners) {
listener(state)
}
}
}
const host: HostProfile = {
id: 'host-1',
name: 'Blue Whale',
endpoint: 'ws://192.168.1.10:6768',
deviceToken: 'device-token',
publicKeyB64: 'A'.repeat(44),
lastConnected: 1,
endpoints: [
{ id: 'lan', kind: 'lan', url: 'ws://192.168.1.10:6768' },
{ id: 'tailscale', kind: 'tailscale', url: 'ws://100.64.0.2:6768' }
]
}
describe('mobile direct endpoint probe', () => {
beforeEach(() => vi.useFakeTimers())
afterEach(() => vi.useRealTimers())
it('uses the first authenticated candidate without waiting for a stale primary', async () => {
const clients = new Map<string, FakeClient>()
const openDirect = vi.fn((endpoint: string) => {
const client = new FakeClient('connecting')
clients.set(endpoint, client)
if (endpoint.includes('100.64.0.2')) {
setTimeout(() => client.publishState('connected'), 100)
}
return client
})
const probing = openAuthenticatedDirectEndpoint(host, openDirect, 12_000)
await vi.advanceTimersByTimeAsync(100)
const result = await probing
expect(result?.path).toBe('tailscale')
expect(openDirect).toHaveBeenCalledTimes(2)
expect(clients.get(host.endpoint)?.close).toHaveBeenCalledOnce()
expect(result?.client.close).not.toHaveBeenCalled()
})
})

53
mobile/src/transport/mobile-direct-endpoint-probe.ts
View File

@ -58,14 +58,49 @@ export async function openAuthenticatedDirectEndpoint(
openDirect: (endpoint: string) => RpcClient,
timeoutMs: number
): Promise<{ client: RpcClient; path: Exclude<MobileConnectionPath, 'relay'> } | null> {
for (const endpoint of directEndpointUrls(host)) {
const client = openDirect(endpoint)
try {
await waitForAuthenticatedSession(client, timeoutMs)
return { client, path: directPathForEndpoint(host, endpoint) }
} catch {
client.close()
const endpoints = directEndpointUrls(host)
return await new Promise((resolve) => {
const clients = new Set<RpcClient>()
let remaining = endpoints.length
let settled = false
const rejectCandidate = (): void => {
remaining--
if (!settled && remaining === 0) {
settled = true
resolve(null)
}
}
}
return null
for (const endpoint of endpoints) {
let client: RpcClient
try {
client = openDirect(endpoint)
} catch {
rejectCandidate()
continue
}
clients.add(client)
void waitForAuthenticatedSession(client, timeoutMs).then(
() => {
if (settled) {
client.close()
return
}
settled = true
for (const candidate of clients) {
if (candidate !== client) {
candidate.close()
}
}
resolve({ client, path: directPathForEndpoint(host, endpoint) })
},
() => {
if (settled) {
return
}
client.close()
rejectCandidate()
}
)
}
})
}

11
mobile/src/transport/mobile-endpoint-supervisor.test.ts
View File

@ -578,7 +578,7 @@ describe('mobile endpoint supervisor', () => {
supervisor.stop()
})
it('waits for an external signal instead of polling a host-offline relay', async () => {
it('retries a host-offline relay without requiring an external signal', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4404)))
const deps = dependencies({
@ -591,13 +591,12 @@ describe('mobile endpoint supervisor', () => {
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('disconnected')
expect(vi.getTimerCount()).toBe(0)
supervisor.setForeground(true)
expect(vi.getTimerCount()).toBe(1)
await vi.advanceTimersByTimeAsync(250)
await vi.advanceTimersByTimeAsync(9_999)
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(2)
expect(vi.getTimerCount()).toBe(0)
expect(vi.getTimerCount()).toBe(1)
supervisor.stop()
})

9
mobile/src/transport/mobile-relay-reconnect-controller.test.ts
View File

@ -17,7 +17,7 @@ describe('relay reconnect controller', () => {
vi.useRealTimers()
})
it('keeps one retry timer and cancels it when recovery needs an external signal', () => {
it('keeps one retry timer when recovery changes from capacity to host offline', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
@ -26,11 +26,12 @@ describe('relay reconnect controller', () => {
expect(vi.getTimerCount()).toBe(1)
reconnect.registerFailure(new RelayOuterError(4404))
expect(vi.getTimerCount()).toBe(0)
expect(vi.getTimerCount()).toBe(1)
expect(reconnect.shouldDefer()).toBe(true)
expect(vi.getTimerCount()).toBe(0)
vi.runAllTimers()
vi.advanceTimersByTime(9_999)
expect(onRetry).not.toHaveBeenCalled()
vi.advanceTimersByTime(1)
expect(onRetry).toHaveBeenCalledOnce()
})
it('drops a pending relay retry after direct connectivity wins', () => {

20
mobile/src/transport/mobile-relay-reconnect-controller.ts
View File

@ -15,6 +15,8 @@ const RELAY_BACKOFF_MIN_MS = 250
const RELAY_BACKOFF_BASE_MS = 500
const RELAY_BACKOFF_CEILING_MS = 30_000
const RELAY_STABLE_CONNECTION_MS = RELAY_BACKOFF_CEILING_MS
const RELAY_HOST_OFFLINE_RETRY_MIN_MS = 5_000
const RELAY_HOST_OFFLINE_RETRY_MAX_MS = 15_000
export type RelayReconnectDependencies = {
now: () => number
@ -186,7 +188,8 @@ export class RelayReconnectController {
// only an authenticated relay that survived the stability window resets the streak.
this.activeRelayConnectedAt = null
this.consecutiveFailures += 1
const delay = this.delayMs()
const delay =
recovery?.kind === 'retry-after-host-offline' ? this.hostOfflineDelayMs() : this.delayMs()
this.nextAttemptAt = now + delay
if (error instanceof MobileE2EEAuthenticationError) {
// Why: pairing state cannot change on a timer; polling only wakes the radio.
@ -194,18 +197,16 @@ export class RelayReconnectController {
this.clearTimer()
return
}
if (recovery?.kind === 'wait-for-host-revival') {
// Why: retrying HOST_OFFLINE without a revival signal is polling a known-negative state.
this.recoveryGate = 'external-signal'
this.clearTimer()
return
}
if (recovery?.kind === 'disable-relay-credential') {
// Why: a rejected outer credential cannot recover until direct connectivity refreshes it.
this.recoveryGate = 'fresh-credential'
this.clearTimer()
return
}
if (recovery?.kind === 'retry-after-host-offline') {
// A prior transport timer must not bypass the slower known-offline retry.
this.clearTimer()
}
this.recoveryGate = null
if (!scheduleRetry) {
this.clearTimer()
@ -269,6 +270,11 @@ export class RelayReconnectController {
return Math.max(RELAY_BACKOFF_MIN_MS, Math.floor(cap * this.jitterFraction()))
}
private hostOfflineDelayMs(): number {
const range = RELAY_HOST_OFFLINE_RETRY_MAX_MS - RELAY_HOST_OFFLINE_RETRY_MIN_MS
return RELAY_HOST_OFFLINE_RETRY_MIN_MS + Math.floor(range * this.jitterFraction())
}
private jitterFraction(): number {
const [high, low] = this.dependencies.randomBytes(2)
return (((high ?? 0) << 8) | (low ?? 0)) / 0x1_00_00

7
src/shared/mobile-relay-close-codes.test.ts
View File

@ -42,4 +42,11 @@ describe('mobile relay close-code contract', () => {
directUnaffected: true
})
})
it('keeps host-offline recovery self-healing', () => {
expect(mobileRelayRecoveryFor(4404, 'phone-resume')).toEqual({
kind: 'retry-after-host-offline',
fullJitter: true
})
})
})

4
src/shared/mobile-relay-close-codes.ts
View File

@ -13,7 +13,7 @@ export type MobileRelayLeg = 'host-control' | 'host-data' | 'phone-invite' | 'ph
export type MobileRelayRecovery =
| { kind: 'disable-relay-credential'; directUnaffected: true }
| { kind: 'wait-for-host-revival' }
| { kind: 'retry-after-host-offline'; fullJitter: true }
| { kind: 'reconnect-fresh-e2ee'; fullJitter: true }
| { kind: 'resolve-invite-through-director-ws'; requireStrictlyNewerEpoch: true }
| { kind: 'resolve-resume-through-director-post' }
@ -29,7 +29,7 @@ export function mobileRelayRecoveryFor(
case MOBILE_RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL:
return { kind: 'disable-relay-credential', directUnaffected: true }
case MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE:
return { kind: 'wait-for-host-revival' }
return { kind: 'retry-after-host-offline', fullJitter: true }
case MOBILE_RELAY_CLOSE_CODE.PEER_DROPPED:
return { kind: 'reconnect-fresh-e2ee', fullJitter: true }
case MOBILE_RELAY_CLOSE_CODE.WRONG_CELL: