diff --git a/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts b/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts new file mode 100644 index 000000000..21e7f9aad --- /dev/null +++ b/src/main/runtime/relay/relay-auth-coordinator-recovery.test.ts @@ -0,0 +1,237 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { RelayAuthCoordinator, type RelayAuthContext } from './relay-auth-coordinator' +import { RelayHttpError } from './relay-http-client' + +const context: RelayAuthContext = { + identity: { userId: 'user-1', profileId: 'profile-1', organizationId: 'org-1' }, + accessToken: 'access-1', + relayEntitled: true +} + +afterEach(() => { + vi.useRealTimers() +}) + +describe('RelayAuthCoordinator transient recovery', () => { + it('retries a transient assignment failure and activates without an external event', async () => { + vi.useFakeTimers() + const broker = { closeNow: vi.fn() } + const openBroker = vi + .fn() + .mockRejectedValueOnce(new RelayHttpError('assignment', 500)) + .mockResolvedValueOnce(broker) + const statuses: string[] = [] + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: (status) => statuses.push(status), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + expect(statuses.at(-1)).toBe('offline') + + await vi.advanceTimersByTimeAsync(501) + expect(openBroker).toHaveBeenCalledTimes(2) + expect(coordinator.getActiveBroker()).toBe(broker) + expect(statuses.at(-1)).toBe('registered') + }) + + it('retries when cloud-session refresh fails before identity can be read', async () => { + vi.useFakeTimers() + const broker = { closeNow: vi.fn() } + const readContext = vi + .fn() + .mockRejectedValueOnce(new Error('temporary cloud session refresh failure')) + .mockResolvedValueOnce(context) + const openBroker = vi.fn().mockResolvedValue(broker) + const coordinator = new RelayAuthCoordinator({ + readContext, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(readContext).toHaveBeenCalledOnce() + expect(openBroker).not.toHaveBeenCalled() + + await vi.advanceTimersByTimeAsync(501) + expect(readContext).toHaveBeenCalledTimes(2) + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBe(broker) + }) + + it('backs a sustained outage off to the five-minute jitter cap', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + for (const delayMs of [500, 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 64_000, 128_000]) { + await vi.advanceTimersByTimeAsync(delayMs) + } + expect(openBroker).toHaveBeenCalledTimes(10) + + await vi.advanceTimersByTimeAsync(149_999) + expect(openBroker).toHaveBeenCalledTimes(10) + await vi.advanceTimersByTimeAsync(1) + expect(openBroker).toHaveBeenCalledTimes(11) + + await vi.advanceTimersByTimeAsync(150_000) + expect(openBroker).toHaveBeenCalledTimes(12) + }) + + it('does not retry a permanent authorization response', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new RelayHttpError('token-exchange', 403)) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('cancels a pending retry as soon as demand disappears', async () => { + vi.useFakeTimers() + let demanded = true + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + hasDemand: () => demanded, + openBroker, + onStatus: vi.fn(), + random: () => 0.75 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + demanded = false + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('re-reads demand when the retry fires and stops without opening again', async () => { + vi.useFakeTimers() + let demanded = true + const statuses: string[] = [] + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + hasDemand: () => demanded, + openBroker, + onStatus: (status) => statuses.push(status), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + demanded = false + await vi.advanceTimersByTimeAsync(501) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(statuses.at(-1)).toBe('standby') + }) + + it('re-reads entitlement when the retry fires and stops after removal', async () => { + vi.useFakeTimers() + let current = context + const openBroker = vi.fn().mockRejectedValue(new RelayHttpError('assignment', 500)) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => current, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + current = { ...context, relayEntitled: false } + await vi.advanceTimersByTimeAsync(501) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('cancels a pending retry immediately when the coordinator is fenced', async () => { + vi.useFakeTimers() + const openBroker = vi.fn().mockRejectedValue(new Error('temporary control open failure')) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => context, + openBroker, + onStatus: vi.fn(), + random: () => 0.5 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + coordinator.fenceAndCloseNow() + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledOnce() + expect(coordinator.getActiveBroker()).toBeNull() + }) + + it('does not carry a pending retry across an identity switch', async () => { + vi.useFakeTimers() + let current = context + const broker = { closeNow: vi.fn() } + const openBroker = vi + .fn() + .mockRejectedValueOnce(new Error('temporary control open failure')) + .mockResolvedValueOnce(broker) + const coordinator = new RelayAuthCoordinator({ + readContext: async () => current, + openBroker, + onStatus: vi.fn(), + random: () => 0.75 + }) + + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledOnce() + + current = { + ...context, + identity: { ...context.identity, profileId: 'profile-2' } + } + coordinator.reconcile() + await vi.advanceTimersByTimeAsync(0) + expect(openBroker).toHaveBeenCalledTimes(2) + expect(coordinator.getActiveBroker()).toBe(broker) + await vi.advanceTimersByTimeAsync(120_000) + + expect(openBroker).toHaveBeenCalledTimes(2) + }) +}) diff --git a/src/main/runtime/relay/relay-auth-coordinator.ts b/src/main/runtime/relay/relay-auth-coordinator.ts index f197edcc2..7f97e33dc 100644 --- a/src/main/runtime/relay/relay-auth-coordinator.ts +++ b/src/main/runtime/relay/relay-auth-coordinator.ts @@ -1,4 +1,5 @@ import type { RelayBrokerStatus } from './relay-session-broker' +import { shouldRetryRelayConnectionError } from './relay-http-client' export type RelayAuthIdentity = { userId: string @@ -26,6 +27,7 @@ type RelayAuthCoordinatorOptions = { }) => Promise onStatus: (status: RelayBrokerStatus) => void lingerMs?: number + random?: () => number } type BrokerOwnership = { @@ -39,12 +41,17 @@ function identityKey(identity: RelayAuthIdentity): string { } export class RelayAuthCoordinator { + // Why: recover brief failures quickly without turning a sustained outage into auth/director load. + private static readonly RETRY_BASE_MS = 1_000 + private static readonly RETRY_MAX_MS = 5 * 60_000 private readonly options: RelayAuthCoordinatorOptions private authEpoch = 0 private ownership: BrokerOwnership | null = null private readonly pendingOwnerships = new Set() private latestReconcile: Promise = Promise.resolve() private lingerTimer: ReturnType | null = null + private retryTimer: ReturnType | null = null + private retryAttempt = 0 private stopped = false constructor(options: RelayAuthCoordinatorOptions) { @@ -52,12 +59,20 @@ export class RelayAuthCoordinator { } reconcile(): void { + this.beginReconcile(true) + } + + private beginReconcile(resetRetry: boolean, expectedIdentityKey?: string): void { if (this.stopped) { return } + this.cancelRetry() + if (resetRetry) { + this.retryAttempt = 0 + } const epoch = ++this.authEpoch this.invalidatePendingOwnerships() - const reconcile = this.reconcileEpoch(epoch) + const reconcile = this.reconcileEpoch(epoch, expectedIdentityKey) this.latestReconcile = reconcile void reconcile } @@ -65,6 +80,8 @@ export class RelayAuthCoordinator { fenceAndCloseNow(): void { ++this.authEpoch this.cancelLinger() + this.cancelRetry() + this.retryAttempt = 0 this.invalidatePendingOwnerships() this.invalidateOwnership() this.options.onStatus('offline') @@ -94,7 +111,8 @@ export class RelayAuthCoordinator { this.fenceAndCloseNow() } - private async reconcileEpoch(epoch: number): Promise { + private async reconcileEpoch(epoch: number, expectedIdentityKey?: string): Promise { + let retryIdentityKey: string | undefined try { const context = await this.options.readContext() if (!this.isEpochCurrent(epoch)) { @@ -102,12 +120,19 @@ export class RelayAuthCoordinator { } if (!context || !context.relayEntitled) { this.cancelLinger() + this.retryAttempt = 0 this.invalidateOwnership() this.options.onStatus('offline') return } const nextIdentityKey = identityKey(context.identity) + if (expectedIdentityKey && nextIdentityKey !== expectedIdentityKey) { + this.retryAttempt = 0 + this.options.onStatus('offline') + return + } if (!(this.options.hasDemand?.(context) ?? true)) { + this.retryAttempt = 0 if (this.ownership?.valid && this.ownership.identityKey !== nextIdentityKey) { this.cancelLinger() this.invalidateOwnership() @@ -119,9 +144,11 @@ export class RelayAuthCoordinator { } this.cancelLinger() if (this.ownership?.valid && this.ownership.identityKey === nextIdentityKey) { + this.retryAttempt = 0 this.options.onStatus('registered') return } + retryIdentityKey = nextIdentityKey this.invalidateOwnership() this.options.onStatus('connecting') const ownership: BrokerOwnership = { @@ -150,14 +177,42 @@ export class RelayAuthCoordinator { return } this.ownership = ownership + this.retryAttempt = 0 this.options.onStatus('registered') - } catch { + } catch (error) { if (this.isEpochCurrent(epoch)) { this.options.onStatus('offline') + if (shouldRetryRelayConnectionError(error)) { + this.scheduleRetry(epoch, retryIdentityKey) + } } } } + private scheduleRetry(epoch: number, expectedIdentityKey?: string): void { + if (this.retryTimer || !this.isEpochCurrent(epoch)) { + return + } + const exponent = Math.min( + this.retryAttempt, + Math.ceil(Math.log2(RelayAuthCoordinator.RETRY_MAX_MS / RelayAuthCoordinator.RETRY_BASE_MS)) + ) + const capMs = Math.min( + RelayAuthCoordinator.RETRY_MAX_MS, + RelayAuthCoordinator.RETRY_BASE_MS * 2 ** exponent + ) + this.retryAttempt++ + const random = this.options.random ?? Math.random + const delayMs = Math.floor(random() * (capMs + 1)) + this.retryTimer = setTimeout(() => { + this.retryTimer = null + if (this.isEpochCurrent(epoch)) { + // Retry still re-reads entitlement and demand; the timer grants no authority. + this.beginReconcile(false, expectedIdentityKey) + } + }, delayMs) + } + private async refreshAccessToken( ownership: { valid: boolean }, expectedIdentityKey: string @@ -212,6 +267,13 @@ export class RelayAuthCoordinator { } } + private cancelRetry(): void { + if (this.retryTimer) { + clearTimeout(this.retryTimer) + this.retryTimer = null + } + } + private invalidatePendingOwnerships(): void { for (const ownership of this.pendingOwnerships) { ownership.valid = false diff --git a/src/main/runtime/relay/relay-http-client.ts b/src/main/runtime/relay/relay-http-client.ts index 1906381ce..6862f26ce 100644 --- a/src/main/runtime/relay/relay-http-client.ts +++ b/src/main/runtime/relay/relay-http-client.ts @@ -38,6 +38,18 @@ export class RelayHttpError extends Error { } } +export function shouldRetryRelayConnectionError(error: unknown): boolean { + if (!(error instanceof RelayHttpError)) { + return true + } + return ( + error.statusCode >= 500 || + error.statusCode === 408 || + error.statusCode === 425 || + error.statusCode === 429 + ) +} + export function deriveRelayHostId(publicKey: Uint8Array): string { return createHash('sha256').update(publicKey).digest('base64url').slice(0, 16) }