diff --git a/docs/reference/headless-linux-server.md b/docs/reference/headless-linux-server.md index cdee2a4ca..2284bcdaa 100644 --- a/docs/reference/headless-linux-server.md +++ b/docs/reference/headless-linux-server.md @@ -155,6 +155,8 @@ display `:99` when no display exists: Description=Orca runtime server After=network-online.target Wants=network-online.target +StartLimitIntervalSec=300 +StartLimitBurst=5 [Service] Type=simple @@ -165,6 +167,7 @@ ExecStart=/opt/orca/orca-linux.AppImage serve --port 6768 --pairing-address 100. StandardOutput=journal StandardError=journal Restart=on-failure +RestartPreventExitStatus=3 RestartSec=5 [Install] @@ -174,6 +177,19 @@ WantedBy=multi-user.target Replace `100.64.1.20` with the LAN, Tailscale, tunnel, or public hostname that clients should use. +Exit status `3` means another process already owns this userData profile, so +`RestartPreventExitStatus=3` stops the unit instead of retrying a launch that +cannot succeed. Any other permanent startup fault is capped at 5 starts per +5 minutes; systemd's defaults (10s window, 5 starts) can never trip at +`RestartSec=5`, which is how one bad launch could restart thousands of times. +The start limit counts operator-initiated starts too, so once it trips systemd +refuses a plain `systemctl start` until the 5-minute window rolls over. Run +`sudo systemctl reset-failed orca-serve.service` first to clear it — the +[Upgrade](#upgrade-steps) and [Roll back](#roll-back) scripts already do. +On systemd older than 230 those two directives are spelled +`StartLimitInterval=`/`StartLimitBurst=` and belong in `[Service]`; Ubuntu +20.04, Orca's oldest supported base, ships systemd 245. + Enable the service: ```bash @@ -228,6 +244,8 @@ Then add the display dependency to the Orca service: Description=Orca runtime server After=network-online.target orca-xvfb.service Wants=network-online.target orca-xvfb.service +StartLimitIntervalSec=300 +StartLimitBurst=5 [Service] Type=simple @@ -237,6 +255,7 @@ Environment=DISPLAY=:99 Environment=LIBGL_ALWAYS_SOFTWARE=1 ExecStart=/opt/orca/orca-linux.AppImage serve --port 6768 --pairing-address 100.64.1.20 Restart=on-failure +RestartPreventExitStatus=3 RestartSec=5 [Install] @@ -415,6 +434,8 @@ recover_failed_upgrade() { sudo rm -f /opt/orca/orca-linux.AppImage.recovering \ /opt/orca/VERSION.recovering if ((recovery_ok)); then + # A tripped StartLimitBurst refuses a plain start + sudo systemctl reset-failed orca-serve.service || true sudo systemctl start orca-serve.service || true else echo 'Upgrade recovery failed; service remains stopped' >&2 @@ -482,6 +503,8 @@ sudo mv "$ORCA_ROLLBACK_NEW" "$ORCA_ROLLBACK" ORCA_BINARY_PROMOTED=1 sudo mv -f /opt/orca/orca-linux.AppImage.new /opt/orca/orca-linux.AppImage sudo mv -f /opt/orca/VERSION.new /opt/orca/VERSION +# Clears a start-limit hit left by the version being replaced +sudo systemctl reset-failed orca-serve.service sudo systemctl start orca-serve.service ORCA_SERVICE_STOPPED=0 trap - EXIT @@ -614,6 +637,8 @@ restart_after_rollback_error() { fi fi if ((recovery_ok)); then + # A tripped StartLimitBurst refuses a plain start + sudo systemctl reset-failed orca-serve.service || true sudo systemctl start orca-serve.service || true else echo 'Rollback recovery failed; service remains stopped' >&2 @@ -714,6 +739,8 @@ if ((ORCA_ROLLBACK_HAS_VERSION)); then else sudo rm -f /opt/orca/VERSION fi +# The crash-looping build you are rolling back from tripped StartLimitBurst +sudo systemctl reset-failed orca-serve.service sudo systemctl start orca-serve.service ORCA_SERVICE_STOPPED=0 sudo rm -rf -- "$ORCA_RESTORE" @@ -805,9 +832,24 @@ refuse to run there and print the command to run on the machine you want. `orca` user and that `/opt/orca` is readable by that user. - Clients cannot connect: make sure `--pairing-address` is an address reachable from the client, and make sure firewalls allow the selected `--port`. +- Journal shows `Another Orca instance is already running for this userData + profile` and the unit exits `3`: another process already owns the profile, so + `RestartPreventExitStatus=3` leaves the unit `failed` on purpose. Find the + owner with `systemctl status orca-serve` and `pgrep -af orca`. Stop it (or + keep it and leave the unit down), then run + `sudo systemctl reset-failed orca-serve && sudo systemctl start orca-serve` — + `reset-failed` clears the failed state and any start-limit counter. If no owner + exists, the lock is stale (Chromium recorded a pid that + has since been reused): remove `SingletonLock` and `SingletonSocket` from the + userData directory and start again. If an earlier crash-loop already leaked + AppImage mounts, list them with `findmnt -rn -t fuse.orca-linux.AppImage` and + release only the ones with no live owner using `fusermount -uz ` (or + `umount -l `), leaving the running instance's mount alone. - Service crash-loops right after an upgrade: use [Roll back](#roll-back) with the pre-upgrade `.ready` bundle. Do not rerun the upgrade first; doing so would - make the crashing version the next rollback binary. + make the crashing version the next rollback binary. The loop trips + `StartLimitBurst`, so any manual `systemctl start` outside that script needs + `sudo systemctl reset-failed orca-serve.service` first. - Diagnosing other missing libraries: extract the AppImage without launching it with `./orca-linux.AppImage --appimage-extract`, then run `ldd squashfs-root/orca` to list any shared libraries the host is missing. diff --git a/src/main/index.ts b/src/main/index.ts index 39fc076be..7ed25c33d 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -160,8 +160,10 @@ import { acquireSingleInstanceLock, logSingleInstanceLockBypass, logSingleInstanceLockFailure, + shouldActivateDesktopForSecondInstance, shouldBypassSingleInstanceLock, - shouldSkipSingleInstanceLock + shouldSkipSingleInstanceLock, + SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from './startup/single-instance-lock' import { startEventLoopStallProbe } from './startup/event-loop-stall-probe' import { startMainThreadChurnProbe } from './diagnostics/main-thread-churn-probe' @@ -606,7 +608,11 @@ function focusExistingWindow(): void { }) } -function requestDesktopActivation(): void { +function requestDesktopActivation(argv: readonly string[] = []): void { + // Why: a duplicate `orca serve` must not drag a headless server into opening a desktop window (#11935). + if (!shouldActivateDesktopForSecondInstance(argv)) { + return + } desktopActivationGate.requestActivation() } @@ -726,10 +732,11 @@ if (startupDiagnosticsEnabled) { if (!hasSingleInstanceLock) { // Why: a false-negative lock loss otherwise looks like a silent crash on packaged macOS; `open --stderr` can capture this line. logSingleInstanceLockFailure() - app.quit() + // Why: a graceful quit is deferred pre-ready, so this launch would still walk into Linux display init and SIGSEGV (#11935). + app.exit(SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE) } -// Why: when another process holds the lock we've already quit; skip file-writing side effects so this transient process never touches userData. +// Why: when another process holds the lock we've already exited; skip file-writing side effects so this transient process never touches userData. if (hasSingleInstanceLock) { // Why: couple to dev-parent only for electron-vite desktop runs; `orca serve`'s parent (CLI shim/background shell) isn't the intended server lifetime. const shouldCoupleToDevParent = is.dev && !isServeMode diff --git a/src/main/startup/single-instance-lock-exit.electron.test.ts b/src/main/startup/single-instance-lock-exit.electron.test.ts new file mode 100644 index 000000000..7cf61a4af --- /dev/null +++ b/src/main/startup/single-instance-lock-exit.electron.test.ts @@ -0,0 +1,115 @@ +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { createRequire } from 'node:module' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterAll, describe, expect, it } from 'vitest' +import { SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from './single-instance-lock' + +// Why #11935: the lock-loss gate runs before Electron `ready`, where `app.quit()` is deferred, so a +// duplicate headless `orca serve` kept executing the rest of startup, reached Linux Ozone/X11 init +// with no display, died with SIGSEGV, and systemd restarted it until the leaked AppImage FUSE mounts +// hit the kernel's 1000-mount ceiling. This runs the gate's own termination statement, lifted out of +// `src/main/index.ts`, under the real Electron binary. +// +// Why not a live lock race: Chromium's Linux ProcessSingleton only answers a second process once the +// browser IO thread is up, which needs `ready` and therefore a display. On a display-less CI runner +// the "owner" is treated as stale and the duplicate takes the lock, so the race cannot be staged +// there. Lock acquisition and argv forwarding are covered in `single-instance-lock.test.ts`; what +// only a real process can settle is what the loser does next, which is what this file pins. + +const electronBinary = createRequire(import.meta.url)('electron') as string +const LOCK_LOST = 'LOCK_LOST' +const CONTINUED_INTO_STARTUP = 'CONTINUED_INTO_STARTUP' +const REACHED_TAIL = 'REACHED_TAIL' +const MARKER_ENV = 'ORCA_LOCK_FIXTURE_MARKER' + +const fixtureRoots: string[] = [] + +afterAll(() => { + for (const root of fixtureRoots) { + rmSync(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }) + } +}) + +/** The `app.*` call the shipped lock-loss gate executes, so a revert to `app.quit()` fails here. */ +function readLockLossTermination(): string { + const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8') + const start = source.indexOf('if (!hasSingleInstanceLock) {') + expect(start).toBeGreaterThanOrEqual(0) + const end = source.indexOf('\n}', start) + expect(end).toBeGreaterThan(start) + + return source + .slice(start, end) + .split('\n') + .map((line) => line.trim()) + .filter((line) => line.startsWith('app.')) + .join('\n') +} + +function buildFixtureMain(termination: string): string { + return [ + `const { app } = require('electron')`, + `const { appendFileSync } = require('node:fs')`, + // Why: the marker path travels by env — Chromium rewrites argv before the main script sees it. + `const marker = process.env.${MARKER_ENV}`, + `const mark = (name) => appendFileSync(marker, name + '\\n')`, + `const SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE = ${SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE}`, + `mark('${LOCK_LOST}')`, + termination, + `mark('${CONTINUED_INTO_STARTUP}')`, + // Why: stand in for the rest of `src/main/index.ts`, which on the reported host was display init. + `mark('${REACHED_TAIL}')`, + `process.exit(0)` + ].join('\n') +} + +type FixtureRun = { status: number | null; markers: string[] } + +function runLockLossGate(termination: string): FixtureRun { + const root = mkdtempSync(join(tmpdir(), 'orca-lock-loss-')) + fixtureRoots.push(root) + const dir = join(root, 'fixture') + const marker = join(root, 'markers.log') + mkdirSync(dir, { recursive: true }) + writeFileSync( + join(dir, 'package.json'), + '{ "name": "orca-lock-loss-fixture", "main": "main.js" }' + ) + writeFileSync(join(dir, 'main.js'), buildFixtureMain(termination)) + writeFileSync(marker, '') + + const result = spawnSync(electronBinary, [dir, '--no-sandbox'], { + stdio: 'ignore', + timeout: 60_000, + env: { ...process.env, [MARKER_ENV]: marker } + }) + expect(result.error).toBeUndefined() + + return { + status: result.status, + markers: readFileSync(marker, 'utf8').split('\n').filter(Boolean) + } +} + +describe('#11935 pre-ready lock-loss termination under real Electron', () => { + it('stops the duplicate launch before any further startup runs, with the already-running code', () => { + const termination = readLockLossTermination() + // Why: an empty slice would let the fixture fall through to its own exit and pass vacuously. + expect(termination).not.toBe('') + + const run = runLockLossGate(termination) + + expect(run.markers).toEqual([LOCK_LOST]) + expect(run.status).toBe(SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE) + }, 90_000) + + it('reproduces the deferred graceful quit that let the doomed launch keep booting', () => { + const run = runLockLossGate('app.quit()') + + // Why: pins the Electron semantic the fix rests on — pre-`ready` `quit()` schedules, it does not stop. + expect(run.markers).toEqual([LOCK_LOST, CONTINUED_INTO_STARTUP, REACHED_TAIL]) + expect(run.status).not.toBe(SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE) + }, 90_000) +}) diff --git a/src/main/startup/single-instance-lock-headless-exit.test.ts b/src/main/startup/single-instance-lock-headless-exit.test.ts new file mode 100644 index 000000000..3c9468f59 --- /dev/null +++ b/src/main/startup/single-instance-lock-headless-exit.test.ts @@ -0,0 +1,84 @@ +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { describe, expect, it } from 'vitest' +import { SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE } from './single-instance-lock' + +// Why #11935: a pre-`ready` graceful quit is deferred, so a lock-losing headless `orca serve` +// kept booting into Linux Ozone/X11 init, died with SIGSEGV, and systemd restarted it forever +// until the leaked AppImage FUSE mounts hit the kernel's 1000-mount ceiling. + +function readSystemdUnitBlocks(doc: string): Map { + const blocks = new Map() + // Why: key on the unit's path comment — splitting on directives mixes `[Unit]` and `[Service]` across blocks. + for (const match of doc.matchAll(/^# \/etc\/systemd\/system\/(\S+\.service)$/gm)) { + const start = match.index + match[0].length + const name = match[1] + blocks.set(name, [...(blocks.get(name) ?? []), doc.slice(start, doc.indexOf('```', start))]) + } + return blocks +} + +describe('headless lock-loss exit contract', () => { + const source = readFileSync(join(process.cwd(), 'src/main/index.ts'), 'utf8') + const doc = readFileSync(join(process.cwd(), 'docs/reference/headless-linux-server.md'), 'utf8') + + it('exits the lock-losing launch immediately instead of scheduling a graceful quit', () => { + const gateStart = source.indexOf('if (!hasSingleInstanceLock) {') + // Why: bound the anchor — an unresolved indexOf slices to EOF and passes vacuously. + expect(gateStart).toBeGreaterThanOrEqual(0) + const gateEnd = source.indexOf('\n}', gateStart) + expect(gateEnd).toBeGreaterThan(gateStart) + + const gate = source.slice(gateStart, gateEnd) + expect(gate).toContain('app.exit(SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE)') + expect(gate).not.toContain('app.quit()') + }) + + it('keeps a duplicate serve launch from promoting the live server to a desktop window', () => { + const activationStart = source.indexOf('function requestDesktopActivation(') + expect(activationStart).toBeGreaterThanOrEqual(0) + const activationEnd = source.indexOf('\n}', activationStart) + expect(activationEnd).toBeGreaterThan(activationStart) + + expect(source.slice(activationStart, activationEnd)).toContain( + 'shouldActivateDesktopForSecondInstance(argv)' + ) + }) + + it('makes every documented serve unit treat a duplicate owner as terminal', () => { + const serveUnits = readSystemdUnitBlocks(doc).get('orca-serve.service') ?? [] + + expect(serveUnits.length).toBeGreaterThan(0) + for (const unit of serveUnits) { + expect(unit).toContain( + `RestartPreventExitStatus=${SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE}` + ) + expect(unit).toContain('StartLimitIntervalSec=') + expect(unit).toContain('StartLimitBurst=') + } + }) + + it('clears the start limit before every scripted start, which a tripped burst would refuse', () => { + const lines = doc.split('\n') + const startLines = lines.flatMap((line, index) => + /^\s*sudo systemctl start orca-serve/.test(line) ? [index] : [] + ) + + expect(startLines.length).toBeGreaterThan(0) + for (const index of startLines) { + expect(lines.slice(Math.max(0, index - 3), index).join('\n')).toContain( + 'systemctl reset-failed orca-serve' + ) + } + }) + + it('leaves the Xvfb unit free to self-heal from a transient display flap', () => { + const xvfbUnits = readSystemdUnitBlocks(doc).get('orca-xvfb.service') ?? [] + + expect(xvfbUnits.length).toBeGreaterThan(0) + // Why: a start limit here would down the display unit permanently and take orca-serve with it. + for (const unit of xvfbUnits) { + expect(unit).not.toContain('StartLimitBurst=') + } + }) +}) diff --git a/src/main/startup/single-instance-lock.test.ts b/src/main/startup/single-instance-lock.test.ts index 5ccd70e6a..b3058c91a 100644 --- a/src/main/startup/single-instance-lock.test.ts +++ b/src/main/startup/single-instance-lock.test.ts @@ -4,8 +4,10 @@ import { acquireSingleInstanceLock, logSingleInstanceLockBypass, logSingleInstanceLockFailure, + shouldActivateDesktopForSecondInstance, shouldBypassSingleInstanceLock, shouldSkipSingleInstanceLock, + SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE, SINGLE_INSTANCE_LOCK_BYPASS_MESSAGE, SINGLE_INSTANCE_LOCK_FAILURE_MESSAGE } from './single-instance-lock' @@ -56,11 +58,11 @@ describe('acquireSingleInstanceLock', () => { expect(acquired).toBe(true) expect(fake.requestSingleInstanceLock).toHaveBeenCalledTimes(1) expect(fake.on).toHaveBeenCalledTimes(1) - expect(fake.on).toHaveBeenCalledWith('second-instance', onSecondInstance) + expect(fake.on).toHaveBeenCalledWith('second-instance', expect.any(Function)) expect(fake.listeners['second-instance']).toHaveLength(1) }) - it('fires the registered callback when second-instance dispatches', () => { + it('forwards the second launch argv so the owner can decide whether to activate', () => { const onSecondInstance = vi.fn() const fake = makeFakeApp(true) @@ -68,9 +70,30 @@ describe('acquireSingleInstanceLock', () => { const [registered] = fake.listeners['second-instance'] ?? [] expect(registered).toBeDefined() - registered?.() + registered?.({}, ['/opt/orca/orca-linux.AppImage', '--serve'], '/home/orca') expect(onSecondInstance).toHaveBeenCalledTimes(1) + expect(onSecondInstance).toHaveBeenCalledWith(['/opt/orca/orca-linux.AppImage', '--serve']) + }) +}) + +describe('shouldActivateDesktopForSecondInstance', () => { + it('ignores a duplicate serve launch but still activates for a desktop launch', () => { + // Why: a supervisor respawning `orca serve` must not open a window on a display-less host (#11935). + const serveArgv = ['/opt/orca/orca-linux.AppImage', '--serve'] + expect(shouldActivateDesktopForSecondInstance(serveArgv)).toBe(false) + expect(shouldActivateDesktopForSecondInstance(['/Applications/Orca.app/orca'])).toBe(true) + }) + + it('fails open when no argv is available', () => { + expect(shouldActivateDesktopForSecondInstance([])).toBe(true) + expect(shouldActivateDesktopForSecondInstance()).toBe(true) + }) +}) + +describe('SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE', () => { + it('stays 3 because the documented systemd unit keys RestartPreventExitStatus off it', () => { + expect(SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE).toBe(3) }) }) diff --git a/src/main/startup/single-instance-lock.ts b/src/main/startup/single-instance-lock.ts index 078500284..4a08fa61e 100644 --- a/src/main/startup/single-instance-lock.ts +++ b/src/main/startup/single-instance-lock.ts @@ -7,6 +7,17 @@ export const SINGLE_INSTANCE_LOCK_BYPASS_ENV = 'ORCA_BYPASS_SINGLE_INSTANCE_LOCK export const SINGLE_INSTANCE_LOCK_E2E_ENFORCE_ENV = 'ORCA_E2E_ENFORCE_SINGLE_INSTANCE_LOCK' export const SINGLE_INSTANCE_LOCK_BYPASS_MESSAGE = '[single-instance] ORCA_BYPASS_SINGLE_INSTANCE_LOCK=1 is set; bypassing the packaged macOS single-instance lock for diagnostics. Do not use this with another Orca instance running for the same profile.' +// Why: stable "another process owns this profile" contract that systemd RestartPreventExitStatus= keys off; changing it silently un-fixes #11935. +export const SINGLE_INSTANCE_ALREADY_RUNNING_EXIT_CODE = 3 + +// Why: `serve` is a CLI subcommand, never Electron argv — an AppImage launched as `orca serve` exits +// at the CLI redirect before requesting the lock, and the CLI re-spawns the Electron child with `--serve`. +const SERVE_MODE_ARG = '--serve' + +// Why: a duplicate `orca serve` is a supervisor artifact, not a user asking for a window; fail open when argv is unavailable. +export function shouldActivateDesktopForSecondInstance(argv: readonly string[] = []): boolean { + return !argv.includes(SERVE_MODE_ARG) +} /** * Why: Orca writes two canonical discovery files into `/`: @@ -26,11 +37,14 @@ export const SINGLE_INSTANCE_LOCK_BYPASS_MESSAGE = * way dev (`orca-dev` userData) and packaged (`orca` userData) runs lock in * separate namespaces instead of serialising against each other. */ -export function acquireSingleInstanceLock(app: App, onSecondInstance: () => void): boolean { +export function acquireSingleInstanceLock( + app: App, + onSecondInstance: (argv: readonly string[]) => void +): boolean { if (!app.requestSingleInstanceLock()) { return false } - app.on('second-instance', onSecondInstance) + app.on('second-instance', (_event, argv) => onSecondInstance(argv)) return true }