From e1071f59e9b7c8e5e1d2ece693425a8fefb7c0f7 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Tue, 4 Aug 2026 02:02:53 -0700 Subject: [PATCH] Say why an update install failed instead of stalling for three minutes (#12224) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Surfaces the real install-failure cause instead of letting a failed elevation stall silently, and keeps the reconnect wait inside its total budget by recomputing the remaining time after each awaited RPC. Relates to #11906 — this fixes the observability half. The functional half (a .deb/.rpm host cannot elevate and can never self-update) is unchanged, so the issue stays open. --- .../updater.install-failure-cause.test.ts | 333 ++++++++++++++++++ src/main/updater.test.ts | 20 +- src/main/updater.ts | 58 ++- .../components/UpdateCard.error-card.test.tsx | 25 ++ .../settings/RemoteServerUpdateDialog.tsx | 4 +- ...emote-server-install-failure-probe.test.ts | 104 ++++++ .../remote-server-install-failure-probe.ts | 36 ++ .../remote-server-restart-wait.test.ts | 178 ++++++++++ .../src/runtime/remote-server-restart-wait.ts | 81 +++++ .../remote-server-update-coordinator.test.ts | 89 +++++ .../remote-server-update-coordinator.ts | 56 ++- .../src/store/slices/remote-server-updates.ts | 9 +- src/shared/app-version.ts | 10 + 13 files changed, 949 insertions(+), 54 deletions(-) create mode 100644 src/main/updater.install-failure-cause.test.ts create mode 100644 src/renderer/src/runtime/remote-server-install-failure-probe.test.ts create mode 100644 src/renderer/src/runtime/remote-server-install-failure-probe.ts create mode 100644 src/renderer/src/runtime/remote-server-restart-wait.test.ts create mode 100644 src/renderer/src/runtime/remote-server-restart-wait.ts diff --git a/src/main/updater.install-failure-cause.test.ts b/src/main/updater.install-failure-cause.test.ts new file mode 100644 index 000000000..5e61ac195 --- /dev/null +++ b/src/main/updater.install-failure-cause.test.ts @@ -0,0 +1,333 @@ +import os from 'node:os' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type * as TracerModule from './observability/tracer' +import type * as UpdaterModule from './updater' + +const { + appMock, + browserWindowMock, + nativeUpdaterMock, + autoUpdaterMock, + isMock, + killAllPtyMock, + recordUpdaterLifecycleMock +} = vi.hoisted(() => { + const appEventHandlers = new Map void)[]>() + const eventHandlers = new Map void)[]>() + + const appOn = vi.fn((event: string, handler: (...args: unknown[]) => void) => { + const handlers = appEventHandlers.get(event) ?? [] + handlers.push(handler) + appEventHandlers.set(event, handlers) + return appMock + }) + + const on = vi.fn((event: string, handler: (...args: unknown[]) => void) => { + const handlers = eventHandlers.get(event) ?? [] + handlers.push(handler) + eventHandlers.set(event, handlers) + return autoUpdaterMock + }) + + const emit = (event: string, ...args: unknown[]) => { + for (const handler of eventHandlers.get(event) ?? []) { + handler(...args) + } + } + + const reset = () => { + appEventHandlers.clear() + appOn.mockClear() + eventHandlers.clear() + on.mockClear() + autoUpdaterMock.checkForUpdates.mockReset() + autoUpdaterMock.downloadUpdate.mockReset() + autoUpdaterMock.quitAndInstall.mockReset() + autoUpdaterMock.setFeedURL.mockClear() + } + + const autoUpdaterMock = { + autoDownload: false, + autoInstallOnAppQuit: false, + on, + checkForUpdates: vi.fn(), + downloadUpdate: vi.fn(), + quitAndInstall: vi.fn(), + setFeedURL: vi.fn(), + emit, + reset + } + + return { + appMock: { + isPackaged: true, + getVersion: vi.fn(() => '1.4.162'), + on: appOn, + quit: vi.fn(), + exit: vi.fn() + }, + browserWindowMock: { getAllWindows: vi.fn(() => []) }, + nativeUpdaterMock: { on: vi.fn() }, + autoUpdaterMock, + isMock: { dev: false }, + killAllPtyMock: vi.fn(), + recordUpdaterLifecycleMock: vi.fn() + } +}) + +vi.mock('electron', () => ({ + app: appMock, + BrowserWindow: browserWindowMock, + autoUpdater: nativeUpdaterMock, + powerMonitor: { on: vi.fn() }, + shell: { openExternal: vi.fn() }, + net: { fetch: vi.fn() } +})) + +vi.mock('electron-updater', () => ({ autoUpdater: autoUpdaterMock })) +vi.mock('./electron-updater-loader', () => ({ + loadElectronAutoUpdater: () => autoUpdaterMock +})) +vi.mock('@electron-toolkit/utils', () => ({ is: isMock })) +vi.mock('./ipc/pty', () => ({ killAllPty: killAllPtyMock })) +vi.mock('./updater-changelog', () => ({ + fetchChangelog: vi.fn().mockResolvedValue(null) +})) +vi.mock('./updater-nudge', () => ({ + fetchNudge: vi.fn().mockResolvedValue(null), + shouldApplyNudge: vi.fn().mockReturnValue(false) +})) +vi.mock('./updater-lifecycle-diagnostics', () => ({ + recordUpdaterLifecycle: recordUpdaterLifecycleMock +})) + +// The real electron-updater DebUpdater failure text when elevation is impossible. +const DEB_ELEVATION_ERROR = + 'Error: Command failed: /usr/bin/pkexec --disable-internal-agent "/bin/bash" "-c" "dpkg -i \'/home/u/.cache/orca-updater/pending/orca-ide_1.4.163_amd64.deb\'"\npkexec must be setuid root' + +// electron-updater's ERR_UPDATER_INVALID_SIGNATURE text, which drives its own card in UpdateCard. +const WINDOWS_SIGNATURE_MISMATCH_ERROR = + 'New version 1.4.163 is not signed by the application owner: publisherNames: Orca, Inc.' + +type CapturedSpan = { + readonly name: string + readonly exit: { readonly _tag: string; readonly cause?: string } +} + +const originalPlatform = process.platform + +let spans: CapturedSpan[] +let tracer: typeof TracerModule | null = null + +function capturingSink(): TracerModule.TracerSink { + return { + push(record) { + spans.push(record as CapturedSpan) + }, + flush() { + /* no-op */ + }, + close() { + /* no-op */ + } + } +} + +function installSpan(): CapturedSpan | undefined { + return spans.find((span) => span.name === 'updater.install') +} + +/** Drives the updater to `downloaded`, the only state `quitAndInstall` acts on. */ +async function reachDownloaded(): Promise { + const mainWindow = { webContents: { send: vi.fn() } } + autoUpdaterMock.checkForUpdates.mockResolvedValue(undefined) + // Why: resetModules gives each test a fresh module graph, so the sink must be installed on the + // same tracer instance updater.ts will import. + tracer = await import('./observability/tracer') + tracer.setActiveSink(capturingSink()) + const updater = await import('./updater') + + updater.setupAutoUpdater(mainWindow as never) + await vi.waitFor(() => { + expect(autoUpdaterMock.checkForUpdates).toHaveBeenCalledTimes(1) + }) + autoUpdaterMock.emit('checking-for-update') + autoUpdaterMock.emit('update-available', { version: '1.4.163' }) + await new Promise((resolve) => setTimeout(resolve, 0)) + autoUpdaterMock.emit('update-downloaded', { version: '1.4.163' }) + expect(updater.getUpdateStatus().state).toBe('downloaded') + return updater +} + +/** + * On a `.deb` Linux host electron-updater's `install()` catches the failed elevation and + * re-dispatches it through the 'error' event *synchronously* inside `quitAndInstall()`. Orca + * recovers the app state, so the payload has to survive on the status and the span has to exit + * Failure — otherwise the only record of why the install never ran is destroyed (#11906). + */ +describe('quitAndInstall failure carries the updater cause', () => { + beforeEach(() => { + vi.resetModules() + autoUpdaterMock.reset() + nativeUpdaterMock.on.mockReset() + browserWindowMock.getAllWindows.mockReset() + browserWindowMock.getAllWindows.mockReturnValue([]) + appMock.getVersion.mockReset() + appMock.getVersion.mockReturnValue('1.4.162') + appMock.quit.mockReset() + appMock.isPackaged = true + isMock.dev = false + killAllPtyMock.mockReset() + recordUpdaterLifecycleMock.mockReset() + Object.defineProperty(process, 'platform', { + value: 'linux', + configurable: true + }) + vi.useRealTimers() + spans = [] + }) + + afterEach(() => { + Object.defineProperty(process, 'platform', { + value: originalPlatform, + configurable: true + }) + tracer?._resetTracerForTests() + tracer = null + }) + + it('reports the underlying elevation failure rather than a generic restart message', async () => { + const { quitAndInstall, getUpdateStatus } = await reachDownloaded() + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR)) + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1) + }) + + const status = getUpdateStatus() + expect(status.state).toBe('error') + // The only place the real cause exists is this event payload; dropping it leaves nothing + // in the logs and nothing actionable in the client's remote-server update dialog. + const message = status.state === 'error' ? status.message : '' + expect(message).toContain('pkexec') + expect(message).toContain('Could not start the update installer.') + }) + + it('keeps the durable breadcrumb to classification only', async () => { + const { quitAndInstall } = await reachDownloaded() + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR)) + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1) + }) + + expect(recordUpdaterLifecycleMock).toHaveBeenCalledWith( + 'quit_and_install_failed_via_event', + { errorType: 'Error' }, + expect.anything() + ) + }) + + it('exits the install span Failure with the cause', async () => { + const { quitAndInstall } = await reachDownloaded() + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + autoUpdaterMock.emit('error', new Error(DEB_ELEVATION_ERROR)) + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(installSpan()).toBeDefined() + }) + + expect(installSpan()?.exit._tag).toBe('Failure') + expect(installSpan()?.exit.cause).toContain('pkexec') + }) + + it('exits the install span Success when the installer takes over', async () => { + const { quitAndInstall } = await reachDownloaded() + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + // The installer runs and the old process is left to exit; no 'error' comes back. + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(installSpan()).toBeDefined() + }) + + expect(installSpan()?.exit._tag).toBe('Success') + }) + + it('keeps a Windows signature verdict unprefixed so its own card still renders', async () => { + Object.defineProperty(process, 'platform', { + value: 'win32', + configurable: true + }) + const { quitAndInstall, getUpdateStatus } = await reachDownloaded() + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + autoUpdaterMock.emit('error', new Error(WINDOWS_SIGNATURE_MISMATCH_ERROR)) + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1) + }) + + const status = getUpdateStatus() + // Prefixing this would put two contradictory instructions on one card. + expect(status.state === 'error' ? status.message : '').toBe(WINDOWS_SIGNATURE_MISMATCH_ERROR) + }) + + it('redacts the cause the same way the retained-package card does', async () => { + const { quitAndInstall, getUpdateStatus } = await reachDownloaded() + const home = os.homedir() + const escape = String.fromCharCode(27) + + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + autoUpdaterMock.emit( + 'error', + new Error(`${escape}[31mdpkg -i '${home}/.cache/orca-updater/pending/orca.deb'${escape}[0m`) + ) + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(autoUpdaterMock.quitAndInstall).toHaveBeenCalledTimes(1) + }) + + const status = getUpdateStatus() + const message = status.state === 'error' ? status.message : '' + expect(message).not.toContain(home) + expect(message).not.toContain(escape) + expect(message).toContain("/.cache/orca-updater/pending/orca.deb'") + }) + + it('carries the cause when quitAndInstall throws instead of dispatching an error event', async () => { + const { quitAndInstall, getUpdateStatus } = await reachDownloaded() + + // Squirrel/NSIS can reject the request by throwing out of the native call; the event path never runs. + autoUpdaterMock.quitAndInstall.mockImplementation(() => { + throw new Error('Squirrel.framework is missing from the app bundle') + }) + + quitAndInstall() + await vi.waitFor(() => { + expect(getUpdateStatus().state).toBe('error') + }) + + const status = getUpdateStatus() + const message = status.state === 'error' ? status.message : '' + expect(message).toContain('Squirrel.framework is missing from the app bundle') + expect(message).toContain('Could not start the update installer.') + }) +}) diff --git a/src/main/updater.test.ts b/src/main/updater.test.ts index a0ab47934..dce8b39a7 100644 --- a/src/main/updater.test.ts +++ b/src/main/updater.test.ts @@ -159,6 +159,12 @@ const { getLinuxRootPackageTypeMock, recordUpdaterLifecycleMock } = vi.hoisted(( recordUpdaterLifecycleMock: vi.fn() })) +// Why: macOS keeps the restart advice because quitting does re-stage a Squirrel update. +const PRE_COMMIT_INSTALL_FAILURE = + process.platform === 'darwin' + ? 'Could not restart to install the update. Quit and reopen Orca, then try again.' + : 'Could not start the update installer. Orca remains open.' + // Why: only the marker resolver is faked so the real artifact capture/redaction path stays under test. vi.mock('./linux-update-package-type', () => ({ getLinuxRootPackageType: getLinuxRootPackageTypeMock @@ -1829,10 +1835,8 @@ describe('updater', () => { state: 'error', // Why: a pre-commit install failure is not fixed by restarting, so the copy must not // suggest it — except on macOS, where quitting does re-stage a Squirrel update. - message: - process.platform === 'darwin' - ? 'Could not restart to install the update. Quit and reopen Orca, then try again.' - : 'Could not start the update installer. Orca remains open.' + // The updater's own text is appended because it is the only record of why the install never ran. + message: `${PRE_COMMIT_INSTALL_FAILURE} (No update filepath provided, can't quit and install)` }) ) }) @@ -3911,11 +3915,7 @@ describe('updater', () => { const lastStatus = (send: ReturnType): UpdateStatus | undefined => send.mock.calls.findLast(([channel]) => channel === 'updater:status')?.[1] - // Why: macOS keeps the restart advice because quitting does re-stage a Squirrel update. - const PRE_COMMIT_FAILURE_MESSAGE = - process.platform === 'darwin' - ? 'Could not restart to install the update. Quit and reopen Orca, then try again.' - : 'Could not start the update installer. Orca remains open.' + const PRE_COMMIT_FAILURE_MESSAGE = PRE_COMMIT_INSTALL_FAILURE const AGENT_STDERR = 'pkexec: Error executing command as another user: No authentication agent found.' @@ -4081,7 +4081,7 @@ describe('updater', () => { expect(send).toHaveBeenCalledWith('updater:status', { state: 'error', - message: PRE_COMMIT_FAILURE_MESSAGE + message: `${PRE_COMMIT_FAILURE_MESSAGE} (${EXIT_127})` }) expect(recordUpdaterLifecycleMock).not.toHaveBeenCalledWith( 'linux_package_install_failed', diff --git a/src/main/updater.ts b/src/main/updater.ts index 302591286..96bf163e5 100644 --- a/src/main/updater.ts +++ b/src/main/updater.ts @@ -13,7 +13,10 @@ import type { RemoteServerUpdaterSnapshot, RemoteServerUpdateSupport } from '../shared/remote-server-update' -import { isWindowsSignatureCheckUnavailableFailure } from '../shared/updater-windows-signature-check' +import { + isWindowsSignatureCheckUnavailableFailure, + isWindowsSignatureMismatchFailure +} from '../shared/updater-windows-signature-check' import { killAllPty } from './ipc/pty' import { withUpdaterSpan } from './observability/instrumentation' import { loadElectronAutoUpdater, type ElectronAutoUpdater } from './electron-updater-loader' @@ -763,6 +766,8 @@ async function performQuitAndInstall(): Promise { true ) resetQuitForUpdateState() + // Why: a bare return would exit this span Success and hide the aborted install from tracing. + span.fail('Could not persist the supervised serve update handoff') return } @@ -790,6 +795,12 @@ async function performQuitAndInstall(): Promise { // Why: quitAndInstall can synchronously clear quitAndInstallInProgress via recovery (Win/Linux dispatchError); skip destructive prep if it already ran. if (!quitAndInstallInProgress) { + // Why: recovery already wrote the reason to currentStatus; a bare return would exit this span Success. + span.fail( + currentStatus.state === 'error' + ? currentStatus.message + : 'quitAndInstall returned without invoking the installer' + ) return } @@ -853,8 +864,9 @@ async function performQuitAndInstall(): Promise { recoveryStatus ?? { state: 'error', // Why: past the native invoke this is the same pre-commit failure the event path reports, so it gets the same copy; only a pre-native exception can be helped by a restart. + // A synchronous throw out of quitAndInstall carries the same installer text the 'error' event would have. message: quitAndInstallNativeInvokedBeforeReset - ? getPreCommitInstallFailureMessage() + ? withInstallFailureCause(getPreCommitInstallFailureMessage(), error) : 'Could not restart to install the update. Quit and reopen Orca, then try again.' } ) @@ -890,6 +902,33 @@ function sendInstallFailureStatus(status: UpdateStatus): void { sendStatus(status, { force: true }) } +const INSTALL_FAILURE_CAUSE_MAX_LENGTH = 200 + +/** + * Appends the updater's own text to the generic install-failure copy. Without it the only record of + * why the install never started is destroyed — on Linux that text carries the exact `dpkg -i ` + * command the user has to run by hand, and remote clients get nothing but "it didn't come back". + */ +function withInstallFailureCause(baseMessage: string, error: unknown): string { + const raw = error instanceof Error ? error.message : typeof error === 'string' ? error : '' + // Why: the retained-package card runs its text through this same sanitizer, so a home directory, + // user name, or terminal escape must not reach the card merely because no artifact was tracked. + const redacted = + redactLinuxPackageInstallText(raw, getTrackedLinuxPackageArtifact()?.path ?? null) ?? '' + const cause = redacted.slice(0, INSTALL_FAILURE_CAUSE_MAX_LENGTH) + if (!cause || cause === 'Unknown error') { + return baseMessage + } + // Why: UpdateCard picks the whole card off this string, so a signature verdict must not be prefixed by contradictory restart advice. + if ( + isWindowsSignatureCheckUnavailableFailure(cause) || + isWindowsSignatureMismatchFailure(cause) + ) { + return cause + } + return `${baseMessage} (${cause})` +} + /** * The recovery status for a failed `.deb`/`.rpm` install, or null when no retained package can * recover it. Must run before `resetQuitForUpdateState()` clears the attempt diagnostic. @@ -945,14 +984,19 @@ function handleQuitAndInstallFailure(error?: unknown): boolean { const recoveryStatus = buildLinuxPackageInstallFailureStatus(error) failServeUpdateHandoff('The native updater rejected the install request.') resetQuitForUpdateState() - recordUpdaterLifecycle('quit_and_install_failed_via_event', undefined, { - level: 'warn', - message: 'Update install could not start; recovered app state' - }) + // Durable data carries classification only — the cause text stays on the status the user can read. + recordUpdaterLifecycle( + 'quit_and_install_failed_via_event', + { errorType: error instanceof Error ? error.name : typeof error }, + { + level: 'warn', + message: 'Update install could not start; recovered app state' + } + ) sendInstallFailureStatus( recoveryStatus ?? { state: 'error', - message: getPreCommitInstallFailureMessage() + message: withInstallFailureCause(getPreCommitInstallFailureMessage(), error) } ) return true diff --git a/src/renderer/src/components/UpdateCard.error-card.test.tsx b/src/renderer/src/components/UpdateCard.error-card.test.tsx index 0e41c46f0..4e4aec4d2 100644 --- a/src/renderer/src/components/UpdateCard.error-card.test.tsx +++ b/src/renderer/src/components/UpdateCard.error-card.test.tsx @@ -127,6 +127,19 @@ describe('UpdateCard Windows signature failures', () => { 'true' ) }) + + // An install failure now carries the updater's own text, so it can reach these branches too. + it('routes a signature verdict raised during install to the security-stop card', () => { + const message = + 'New version 1.4.200 is not signed by the application owner: publisherNames: Orca' + renderAfterAvailableStatus() + + act(() => useAppStore.getState().setUpdateStatus({ state: 'error', message })) + + expect(screen.getByText("Update Wasn't Installed")).toBeTruthy() + // The generic restart advice must not be prefixed onto a security stop. + expect(screen.queryByText(/Quit and reopen Orca/)).toBeNull() + }) }) describe('UpdateCard hourly builds', () => { @@ -260,6 +273,18 @@ describe('UpdateCard Linux package-install recovery', () => { expect(download).toHaveBeenCalledTimes(1) }) + it('shows the appended install cause behind the generic card details', () => { + const message = + 'Could not start the update installer. Orca remains open. (Command failed: pkexec must be setuid root)' + renderAfterAvailableStatus() + + act(() => useAppStore.getState().setUpdateStatus({ state: 'error', message })) + + expect(screen.getByText('Update Error')).toBeTruthy() + fireEvent.click(screen.getByRole('button', { name: 'Show details' })) + expect(screen.getByText(message)).toBeTruthy() + }) + it('leaves the HTTP/1.1 compatibility branch untouched', () => { renderAfterAvailableStatus() diff --git a/src/renderer/src/components/settings/RemoteServerUpdateDialog.tsx b/src/renderer/src/components/settings/RemoteServerUpdateDialog.tsx index 918eec334..473caf85f 100644 --- a/src/renderer/src/components/settings/RemoteServerUpdateDialog.tsx +++ b/src/renderer/src/components/settings/RemoteServerUpdateDialog.tsx @@ -93,7 +93,9 @@ function ServerUpdateRow({ {help ? (

{help} diff --git a/src/renderer/src/runtime/remote-server-install-failure-probe.test.ts b/src/renderer/src/runtime/remote-server-install-failure-probe.test.ts new file mode 100644 index 000000000..1c840aecb --- /dev/null +++ b/src/renderer/src/runtime/remote-server-install-failure-probe.test.ts @@ -0,0 +1,104 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + RemoteServerUpdateInstallResult, + RemoteServerUpdaterSnapshot +} from '../../../shared/remote-server-update' +import type { RuntimeStatus } from '../../../shared/runtime-types' +import { readRemoteServerInstallFailure } from './remote-server-install-failure-probe' + +const support = { + installMode: 'supervised-headless-serve', + automatic: true, + reason: 'available' +} as const + +const install: RemoteServerUpdateInstallResult = { + accepted: true, + fromVersion: '1.4.0', + targetVersion: '1.5.0', + runtimeId: 'runtime-old' +} + +function runtime(runtimeId = 'runtime-old'): RuntimeStatus { + return { + runtimeId, + rendererGraphEpoch: 0, + graphStatus: 'ready', + authoritativeWindowId: null, + liveTabCount: 2, + liveLeafCount: 1, + capabilities: ['updater.remote-control.v1'], + appVersion: '1.4.0', + remoteUpdateSupport: support + } +} + +function snapshot( + status: RemoteServerUpdaterSnapshot['status'], + runtimeId = 'runtime-old' +): RemoteServerUpdaterSnapshot { + return { appVersion: '1.4.0', runtimeId, support, status } +} + +describe('readRemoteServerInstallFailure', () => { + it('returns the message the original process is publishing', async () => { + const getUpdaterStatus = vi.fn(async () => + snapshot({ state: 'error', message: 'pkexec must be setuid root' }) + ) + + await expect( + readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime()) + ).resolves.toBe('pkexec must be setuid root') + }) + + it('bounds the status call by the deadline the caller passes', async () => { + const getUpdaterStatus = vi.fn(async () => + snapshot({ state: 'error', message: 'pkexec must be setuid root' }) + ) + + await readRemoteServerInstallFailure( + 'server-1', + { getUpdaterStatus }, + install, + runtime(), + 4_000 + ) + + // Without this the call inherits the RPC client's 15s default and can outlast the reconnect budget. + expect(getUpdaterStatus).toHaveBeenCalledWith('server-1', 4_000) + }) + + it('ignores an error published by a different process', async () => { + const getUpdaterStatus = vi.fn(async () => + snapshot({ state: 'error', message: 'pkexec must be setuid root' }, 'runtime-new') + ) + + await expect( + readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime()) + ).resolves.toBeNull() + }) + + it('ignores a snapshot that is not an error', async () => { + const getUpdaterStatus = vi.fn(async () => snapshot({ state: 'downloaded', version: '1.5.0' })) + + await expect( + readRemoteServerInstallFailure('server-1', { getUpdaterStatus }, install, runtime()) + ).resolves.toBeNull() + }) + + it('costs no round trip once the replacement runtime answers', async () => { + const getUpdaterStatus = vi.fn(async () => + snapshot({ state: 'error', message: 'pkexec must be setuid root' }) + ) + + await expect( + readRemoteServerInstallFailure( + 'server-1', + { getUpdaterStatus }, + install, + runtime('runtime-new') + ) + ).resolves.toBeNull() + expect(getUpdaterStatus).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/runtime/remote-server-install-failure-probe.ts b/src/renderer/src/runtime/remote-server-install-failure-probe.ts new file mode 100644 index 000000000..7f37fd384 --- /dev/null +++ b/src/renderer/src/runtime/remote-server-install-failure-probe.ts @@ -0,0 +1,36 @@ +import type { + RemoteServerUpdateInstallResult, + RemoteServerUpdaterSnapshot +} from '../../../shared/remote-server-update' +import type { RuntimeStatus } from '../../../shared/runtime-types' + +type InstallFailureProbeTransport = { + getUpdaterStatus: ( + environmentId: string, + timeoutMs?: number + ) => Promise +} + +/** + * The install RPC is accepted before the installer runs, so a server that failed to restart keeps + * publishing the reason through `updater.getStatus`. Reading it turns a three-minute reconnect + * timeout into the actual cause. Both runtime-id guards matter: a snapshot from a process that is + * not the one we asked to install describes some other attempt. + */ +export async function readRemoteServerInstallFailure( + environmentId: string, + transport: InstallFailureProbeTransport, + install: RemoteServerUpdateInstallResult, + runtime: RuntimeStatus, + timeoutMs?: number +): Promise { + if (runtime.runtimeId !== install.runtimeId) { + return null + } + // Why: without an explicit deadline this inherits the RPC client's 15s default and can outlast the caller's own wait. + const snapshot = await transport.getUpdaterStatus(environmentId, timeoutMs) + if (snapshot.runtimeId !== install.runtimeId || snapshot.status.state !== 'error') { + return null + } + return snapshot.status.message +} diff --git a/src/renderer/src/runtime/remote-server-restart-wait.test.ts b/src/renderer/src/runtime/remote-server-restart-wait.test.ts new file mode 100644 index 000000000..76567556d --- /dev/null +++ b/src/renderer/src/runtime/remote-server-restart-wait.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, it, vi } from 'vitest' +import type { + RemoteServerUpdateInstallResult, + RemoteServerUpdaterSnapshot +} from '../../../shared/remote-server-update' +import type { RuntimeStatus } from '../../../shared/runtime-types' +import { waitForReplacementRuntime } from './remote-server-restart-wait' + +const support = { + installMode: 'supervised-headless-serve', + automatic: true, + reason: 'available' +} as const + +const install: RemoteServerUpdateInstallResult = { + accepted: true, + fromVersion: '1.4.0', + targetVersion: '1.5.0', + runtimeId: 'runtime-old' +} + +const idleSnapshot: RemoteServerUpdaterSnapshot = { + appVersion: '1.4.0', + runtimeId: 'runtime-old', + support, + status: { state: 'downloaded', version: '1.5.0' } +} + +function runtime(version: string, runtimeId: string): RuntimeStatus { + return { + runtimeId, + rendererGraphEpoch: 0, + graphStatus: 'ready', + authoritativeWindowId: null, + liveTabCount: 2, + liveLeafCount: 1, + capabilities: ['updater.remote-control.v1'], + appVersion: version, + remoteUpdateSupport: support + } +} + +describe('waitForReplacementRuntime', () => { + it('returns the replacement process once it answers on the target version', async () => { + let clock = 0 + let ticks = 0 + + await expect( + waitForReplacementRuntime( + 'server-1', + { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds + }, + getUpdaterStatus: async () => idleSnapshot, + getRuntimeStatus: async () => { + ticks += 1 + return ticks > 1 ? runtime('1.5.0', 'runtime-new') : runtime('1.4.0', 'runtime-old') + } + }, + install, + { reconnectTimeoutMs: 1_000, pollIntervalMs: 10 } + ) + ).resolves.toMatchObject({ runtimeId: 'runtime-new', appVersion: '1.5.0' }) + }) + + it('never lets a poll RPC outlive the reconnect deadline', async () => { + let clock = 0 + const reconnectTimeoutMs = 25_000 + const pollIntervalMs = 500 + + await expect( + waitForReplacementRuntime( + 'server-1', + { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds + }, + getUpdaterStatus: async () => idleSnapshot, + // A wedged SSH link burns the whole allowance it is given before rejecting. + getRuntimeStatus: async (_environmentId, timeoutMs) => { + clock += timeoutMs ?? 0 + throw new Error('runtime status timed out') + } + }, + install, + { reconnectTimeoutMs, pollIntervalMs } + ) + ).rejects.toThrow('remote_update_reconnect_timeout') + + // A tick allowed its full 10s past the deadline would push this well beyond the budget. + expect(clock).toBeLessThanOrEqual(reconnectTimeoutMs) + }) + + it('keeps the total wait inside the budget when the status RPC eats it before the probe', async () => { + let clock = 0 + const reconnectTimeoutMs = 20_000 + + await expect( + waitForReplacementRuntime( + 'server-1', + { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds + }, + // Both RPCs burn everything they are handed, so a stale budget doubles the tick. + getUpdaterStatus: async (_environmentId, timeoutMs) => { + clock += timeoutMs ?? 0 + return idleSnapshot + }, + getRuntimeStatus: async (_environmentId, timeoutMs) => { + clock += timeoutMs ?? 0 + return runtime('1.4.0', 'runtime-old') + } + }, + install, + { reconnectTimeoutMs, pollIntervalMs: 500 } + ) + ).rejects.toThrow('remote_update_reconnect_timeout') + + expect(clock).toBeLessThanOrEqual(reconnectTimeoutMs) + }) + + it('charges the failure probe only the budget left after the status RPC', async () => { + let clock = 0 + const getUpdaterStatus = vi.fn(async () => idleSnapshot) + + await expect( + waitForReplacementRuntime( + 'server-1', + { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds + }, + getUpdaterStatus, + getRuntimeStatus: async () => { + clock += 2_000 + return runtime('1.4.0', 'runtime-old') + } + }, + install, + { reconnectTimeoutMs: 9_000, pollIntervalMs: 1_000 } + ) + ).rejects.toThrow('remote_update_reconnect_timeout') + + // First probe runs on the second tick: it starts at 3s, the status RPC spends 2s more, + // so 4s of the 9s budget is left — not the 6s the tick began with. + expect(getUpdaterStatus).toHaveBeenNthCalledWith(1, 'server-1', 4_000) + }) + + it('bounds the failure probe by the same remaining budget', async () => { + let clock = 0 + const getUpdaterStatus = vi.fn(async () => idleSnapshot) + + await expect( + waitForReplacementRuntime( + 'server-1', + { + now: () => clock, + wait: async (milliseconds) => { + clock += milliseconds + }, + getUpdaterStatus, + getRuntimeStatus: async () => runtime('1.4.0', 'runtime-old') + }, + install, + { reconnectTimeoutMs: 4_000, pollIntervalMs: 1_000 } + ) + ).rejects.toThrow('remote_update_reconnect_timeout') + + // First tick is the probe's warm-up; the second runs with 3s of the 4s budget left. + expect(getUpdaterStatus).toHaveBeenCalledWith('server-1', 3_000) + }) +}) diff --git a/src/renderer/src/runtime/remote-server-restart-wait.ts b/src/renderer/src/runtime/remote-server-restart-wait.ts new file mode 100644 index 000000000..d78b2170f --- /dev/null +++ b/src/renderer/src/runtime/remote-server-restart-wait.ts @@ -0,0 +1,81 @@ +import { hasReachedAppVersion } from '../../../shared/app-version' +import type { + RemoteServerUpdateInstallResult, + RemoteServerUpdaterSnapshot +} from '../../../shared/remote-server-update' +import type { RuntimeStatus } from '../../../shared/runtime-types' +import { readRemoteServerInstallFailure } from './remote-server-install-failure-probe' + +type RestartWaitTransport = { + getRuntimeStatus: (environmentId: string, timeoutMs?: number) => Promise + getUpdaterStatus: ( + environmentId: string, + timeoutMs?: number + ) => Promise + wait: (milliseconds: number) => Promise + now?: () => number +} + +type RestartWaitTiming = { + reconnectTimeoutMs: number + pollIntervalMs: number +} + +/** Per-call ceiling for the two poll RPCs; each tick also clamps to the time left. */ +const RESTART_WAIT_RPC_TIMEOUT_MS = 10_000 + +/** + * Waits for the replacement process to answer on the target version and returns its runtime status. + * Throws the server's own install-failure text as soon as the *original* process publishes one — + * without that, a server that could not restart is indistinguishable from a slow one until the + * reconnect budget runs out, and the user is told the wrong thing. + */ +export async function waitForReplacementRuntime( + environmentId: string, + transport: RestartWaitTransport, + install: RemoteServerUpdateInstallResult, + timing: RestartWaitTiming +): Promise { + const now = transport.now ?? Date.now + const deadline = now() + timing.reconnectTimeoutMs + // Why: the install RPC returns before the installer fires, so an error on the first tick belongs to an earlier attempt. + let probed = false + while (now() < deadline) { + // Why: an RPC allowed to outlive the deadline drags the whole wait past the budget the caller set. + const rpcTimeoutMs = Math.min(RESTART_WAIT_RPC_TIMEOUT_MS, deadline - now()) + let installFailure: string | null = null + try { + const status = await transport.getRuntimeStatus(environmentId, rpcTimeoutMs) + const version = status.appVersion?.trim() ?? '' + const reachedTarget = hasReachedAppVersion(version, install.targetVersion) + if (status.runtimeId !== install.runtimeId && reachedTarget) { + return status + } + // Why: the status RPC already spent part of rpcTimeoutMs, so the probe re-reads what is left + // rather than starting a second full budget of its own. + const probeTimeoutMs = Math.min(RESTART_WAIT_RPC_TIMEOUT_MS, deadline - now()) + installFailure = + probed && probeTimeoutMs > 0 + ? await readRemoteServerInstallFailure( + environmentId, + transport, + install, + status, + probeTimeoutMs + ) + : null + probed = true + } catch { + // A refused connection is expected while the server process is being replaced. + } + // Why: thrown outside the try so the loop's own catch cannot swallow the reason we came for. + if (installFailure !== null) { + throw new Error(installFailure) + } + const waitMs = Math.min(timing.pollIntervalMs, deadline - now()) + if (waitMs > 0) { + await transport.wait(waitMs) + } + } + throw new Error('remote_update_reconnect_timeout') +} diff --git a/src/renderer/src/runtime/remote-server-update-coordinator.test.ts b/src/renderer/src/runtime/remote-server-update-coordinator.test.ts index 824fdf6fb..66cb646f7 100644 --- a/src/renderer/src/runtime/remote-server-update-coordinator.test.ts +++ b/src/renderer/src/runtime/remote-server-update-coordinator.test.ts @@ -44,6 +44,10 @@ function status(version: string, runtimeId = 'runtime-old', automatic = true): R } } +// The main process's own copy once quitAndInstall fails; it must survive to the client verbatim. +const INSTALL_FAILURE = + "Could not start the update installer. Orca remains open. (Command failed: pkexec dpkg -i '/home/u/.cache/orca-updater/pending/orca-ide_1.5.0_amd64.deb' pkexec must be setuid root)" + const availableSnapshot: RemoteServerUpdaterSnapshot = { appVersion: '1.4.0', runtimeId: 'runtime-old', @@ -275,6 +279,91 @@ describe('remote server update execution', () => { }) }) + it('repeats the install failure a still-running server is publishing', async () => { + const snapshots = [ + availableSnapshot, + { ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } } + ] satisfies RemoteServerUpdaterSnapshot[] + let installed = false + const result = await runRemoteServerUpdate( + availableEntry(), + transport({ + getUpdaterStatus: async () => + installed + ? { ...availableSnapshot, status: { state: 'error', message: INSTALL_FAILURE } } + : (snapshots.shift() ?? availableSnapshot), + install: async (): Promise => { + installed = true + return { + accepted: true, + fromVersion: '1.4.0', + targetVersion: '1.5.0', + runtimeId: 'runtime-old' + } + }, + // The server never restarts: same runtimeId, same version, still perfectly reachable. + getRuntimeStatus: async () => status('1.4.0', 'runtime-old') + }), + () => undefined, + { timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 60, pollIntervalMs: 1 } } + ) + expect(result).toMatchObject({ phase: 'failed', error: INSTALL_FAILURE }) + }) + + it('ignores an updater error that predates the install request', async () => { + const snapshots = [ + availableSnapshot, + { ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } } + ] satisfies RemoteServerUpdaterSnapshot[] + let reconnectTicks = 0 + const result = await runRemoteServerUpdate( + availableEntry(), + transport({ + // A background check that errored inside the 100ms before the installer fires. + getUpdaterStatus: async () => + snapshots.shift() ?? { + ...availableSnapshot, + status: { state: 'error', message: 'a background check failed' } + }, + getRuntimeStatus: async () => { + reconnectTicks += 1 + return reconnectTicks > 1 + ? status('1.5.0', 'runtime-new') + : status('1.4.0', 'runtime-old') + } + }), + () => undefined, + { timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 60, pollIntervalMs: 1 } } + ) + expect(result).toMatchObject({ phase: 'updated', currentVersion: '1.5.0' }) + }) + + it('keeps waiting when the failure probe itself cannot be reached', async () => { + const snapshots = [ + availableSnapshot, + { ...availableSnapshot, status: { state: 'downloaded', version: '1.5.0' } } + ] satisfies RemoteServerUpdaterSnapshot[] + const result = await runRemoteServerUpdate( + availableEntry(), + transport({ + getUpdaterStatus: async () => { + const next = snapshots.shift() + if (!next) { + throw new Error('connection refused') + } + return next + }, + getRuntimeStatus: async () => status('1.4.0', 'runtime-old') + }), + () => undefined, + { timing: { operationTimeoutMs: 10, reconnectTimeoutMs: 6, pollIntervalMs: 1 } } + ) + expect(result).toMatchObject({ + phase: 'failed', + error: 'The server did not reconnect on the updated version.' + }) + }) + it('turns a capability race into manual update guidance', async () => { const result = await runRemoteServerUpdate( availableEntry(), diff --git a/src/renderer/src/runtime/remote-server-update-coordinator.ts b/src/renderer/src/runtime/remote-server-update-coordinator.ts index 77b1b6f2c..196699b6e 100644 --- a/src/renderer/src/runtime/remote-server-update-coordinator.ts +++ b/src/renderer/src/runtime/remote-server-update-coordinator.ts @@ -1,5 +1,6 @@ import { compareAppVersions, + hasReachedAppVersion, isPerfPrereleaseAppVersion, isPrereleaseAppVersion, isValidAppVersion @@ -13,6 +14,7 @@ import type { import type { PublicKnownRuntimeEnvironment } from '../../../shared/runtime-environments' import type { RuntimeStatus } from '../../../shared/runtime-types' import type { UpdateCheckOptions } from '../../../shared/types' +import { waitForReplacementRuntime } from './remote-server-restart-wait' import { remoteServerUpdateErrorMessage } from './remote-server-update-errors' import { pollRemoteServerUpdater } from './remote-server-updater-polling' @@ -45,7 +47,10 @@ export type RemoteServerUpdateEntry = { export type RemoteServerUpdateTransport = { getRuntimeStatus: (environmentId: string, timeoutMs?: number) => Promise - getUpdaterStatus: (environmentId: string) => Promise + getUpdaterStatus: ( + environmentId: string, + timeoutMs?: number + ) => Promise check: ( environmentId: string, options: UpdateCheckOptions @@ -213,12 +218,7 @@ export async function runRemoteServerUpdate( if (available.status.state === 'not-available') { const status = await transport.getRuntimeStatus(entry.environmentId, 10_000) const currentVersion = status.appVersion?.trim() ?? '' - const reachedTarget = - entry.targetVersion !== null && - isValidAppVersion(currentVersion) && - isValidAppVersion(entry.targetVersion) && - compareAppVersions(currentVersion, entry.targetVersion) >= 0 - if (!reachedTarget) { + if (!hasReachedAppVersion(currentVersion, entry.targetVersion)) { throw new Error('remote_update_requested_version_unavailable') } next = { @@ -267,34 +267,22 @@ export async function runRemoteServerUpdate( } onProgress(next) - const now = transport.now ?? Date.now - const reconnectDeadline = now() + timing.reconnectTimeoutMs - while (now() < reconnectDeadline) { - try { - const status = await transport.getRuntimeStatus(entry.environmentId, 10_000) - const version = status.appVersion?.trim() ?? '' - const reachedTarget = - isValidAppVersion(version) && - isValidAppVersion(install.targetVersion) && - compareAppVersions(version, install.targetVersion) >= 0 - if (status.runtimeId !== install.runtimeId && reachedTarget) { - next = { - ...next, - phase: 'updated', - currentVersion: version, - runtimeId: status.runtimeId, - liveTabCount: status.liveTabCount, - liveLeafCount: status.liveLeafCount - } - onProgress(next) - return next - } - } catch { - // A refused connection is expected while the server process is being replaced. - } - await transport.wait(timing.pollIntervalMs) + const replacement = await waitForReplacementRuntime( + entry.environmentId, + transport, + install, + timing + ) + next = { + ...next, + phase: 'updated', + currentVersion: replacement.appVersion?.trim() ?? '', + runtimeId: replacement.runtimeId, + liveTabCount: replacement.liveTabCount, + liveLeafCount: replacement.liveLeafCount } - throw new Error('remote_update_reconnect_timeout') + onProgress(next) + return next } catch (error) { next = { ...next, diff --git a/src/renderer/src/store/slices/remote-server-updates.ts b/src/renderer/src/store/slices/remote-server-updates.ts index 909c05e31..80b8a7370 100644 --- a/src/renderer/src/store/slices/remote-server-updates.ts +++ b/src/renderer/src/store/slices/remote-server-updates.ts @@ -33,8 +33,13 @@ const transport: RemoteServerUpdateTransport = { window.api.runtimeEnvironments .getStatus({ selector: environmentId, timeoutMs }) .then((response) => unwrapRuntimeRpcResult(response)), - getUpdaterStatus: (environmentId) => - callRemoteUpdater(environmentId, 'updater.getStatus'), + getUpdaterStatus: (environmentId, timeoutMs) => + callRemoteUpdater( + environmentId, + 'updater.getStatus', + undefined, + timeoutMs + ), check: (environmentId, options) => callRemoteUpdater(environmentId, 'updater.check', options), download: (environmentId) => diff --git a/src/shared/app-version.ts b/src/shared/app-version.ts index a904b4990..e3c537f45 100644 --- a/src/shared/app-version.ts +++ b/src/shared/app-version.ts @@ -94,3 +94,13 @@ export function compareAppVersions(left: string, right: string): number { return 0 } + +/** True when `current` is at or past `target`. Unparseable or absent versions never qualify. */ +export function hasReachedAppVersion(current: string, target: string | null): boolean { + return ( + target !== null && + isValidAppVersion(current) && + isValidAppVersion(target) && + compareAppVersions(current, target) >= 0 + ) +}