fix(ssh): repair unbuilt relay native deps (#8686)
Co-authored-by: Orca <help@stably.ai> Co-authored-by: Neil <4138956+nwparker@users.noreply.github.com> Co-authored-by: Jinwoo-H <jinwoo0825@gmail.com> Co-authored-by: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com>
This commit is contained in:
parent
1284a00e93
commit
fa85536f3a
|
|
@ -23,6 +23,7 @@ type MockSshClient = {
|
|||
lastExecCommand?: string
|
||||
lastConnectConfig?: unknown
|
||||
exec: (cmd: string, cb: (err: Error | undefined, channel: unknown) => void) => void
|
||||
sftp: (cb: (err: Error | undefined, channel: unknown) => void) => void
|
||||
}
|
||||
let clientInstances: MockSshClient[] = []
|
||||
|
||||
|
|
@ -690,15 +691,15 @@ describe('SshConnection', () => {
|
|||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const outcomePromise = conn
|
||||
.exec('printf ready')
|
||||
.then(() => 'opened')
|
||||
.catch((error: Error) => error.message)
|
||||
const outcomePromise = conn.exec('printf ready').catch((error: Error) => error)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
const outcome = await Promise.race([outcomePromise, Promise.resolve('pending')])
|
||||
|
||||
expect(outcome).toBe('SSH exec channel timed out')
|
||||
expect(outcome).toMatchObject({
|
||||
message: 'SSH exec channel timed out',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
|
|
@ -784,15 +785,54 @@ describe('SshConnection', () => {
|
|||
try {
|
||||
const outcomePromise = conn
|
||||
.exec('printf ready', { signal: controller.signal })
|
||||
.then(() => 'opened')
|
||||
.catch((error: Error) => error.name)
|
||||
.catch((error: Error) => error)
|
||||
|
||||
controller.abort()
|
||||
// Why: a hung socket must not pin the aborted caller for the full 30s
|
||||
// connect timeout — the abort settles at the 5s grace bound instead.
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
await expect(outcomePromise).resolves.toBe('AbortError')
|
||||
await expect(outcomePromise).resolves.toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('drains an exec channel that opens after the abort grace has settled', async () => {
|
||||
const conn = new SshConnection(createTarget(), createCallbacks())
|
||||
await conn.connect()
|
||||
execBehavior = 'pending'
|
||||
const controller = new AbortController()
|
||||
const lateChannel = Object.assign(new EventEmitter(), {
|
||||
close: vi.fn(),
|
||||
resume: vi.fn(),
|
||||
stderr: { resume: vi.fn() }
|
||||
})
|
||||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const outcomePromise = conn
|
||||
.exec('printf ready', { signal: controller.signal })
|
||||
.catch((error: Error) => error)
|
||||
|
||||
controller.abort()
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
const outcome = await outcomePromise
|
||||
expect(outcome).toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
|
||||
pendingExecCallback?.(undefined, lateChannel)
|
||||
|
||||
expect(lateChannel.resume).toHaveBeenCalledTimes(1)
|
||||
expect(lateChannel.stderr.resume).toHaveBeenCalledTimes(1)
|
||||
expect(lateChannel.close).toHaveBeenCalledTimes(1)
|
||||
lateChannel.emit('close')
|
||||
expect(outcome).toMatchObject({ sshChannelCloseConfirmed: true })
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
|
|
@ -837,7 +877,7 @@ describe('SshConnection', () => {
|
|||
const lateChannel = Object.assign(new EventEmitter(), {
|
||||
close: vi.fn(),
|
||||
resume: vi.fn(),
|
||||
stderr: { resume: vi.fn() }
|
||||
stderr: Object.assign(new EventEmitter(), { resume: vi.fn() })
|
||||
})
|
||||
|
||||
const outcomePromise = conn
|
||||
|
|
@ -855,11 +895,41 @@ describe('SshConnection', () => {
|
|||
expect(early).toBe('pending')
|
||||
expect(lateChannel.close).toHaveBeenCalledTimes(1)
|
||||
expect(lateChannel.resume).toHaveBeenCalled()
|
||||
expect(() => lateChannel.emit('error', new Error('late channel teardown'))).not.toThrow()
|
||||
expect(() => lateChannel.stderr.emit('error', new Error('late stderr teardown'))).not.toThrow()
|
||||
|
||||
lateChannel.emit('close')
|
||||
await expect(outcomePromise).resolves.toBe('AbortError')
|
||||
})
|
||||
|
||||
it('removes the late-channel close listener when abort grace expires', async () => {
|
||||
const conn = new SshConnection(createTarget(), createCallbacks())
|
||||
await conn.connect()
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
sftpBehavior = 'pending'
|
||||
const controller = new AbortController()
|
||||
const lateSftp = Object.assign(new EventEmitter(), { end: vi.fn() })
|
||||
|
||||
const outcomePromise = conn
|
||||
.sftp(controller.signal)
|
||||
.then(() => 'opened')
|
||||
.catch((error: Error) => error.name)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
controller.abort()
|
||||
pendingSftpCallback?.(undefined, lateSftp)
|
||||
expect(lateSftp.listenerCount('close')).toBe(1)
|
||||
expect(() => lateSftp.emit('error', new Error('late SFTP teardown'))).not.toThrow()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
await expect(outcomePromise).resolves.toBe('AbortError')
|
||||
expect(lateSftp.listenerCount('close')).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('times out when ssh2 never opens an SFTP channel', async () => {
|
||||
const conn = new SshConnection(createTarget(), createCallbacks())
|
||||
await conn.connect()
|
||||
|
|
@ -867,15 +937,13 @@ describe('SshConnection', () => {
|
|||
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const outcomePromise = conn
|
||||
.sftp()
|
||||
.then(() => 'opened')
|
||||
.catch((error: Error) => error.message)
|
||||
const outcomePromise = conn.sftp().catch((error: Error) => error)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
const outcome = await Promise.race([outcomePromise, Promise.resolve('pending')])
|
||||
|
||||
expect(outcome).toBe('SSH SFTP channel timed out')
|
||||
expect(outcome).toMatchObject({ message: 'SSH SFTP channel timed out' })
|
||||
expect(outcome).not.toHaveProperty('sshChannelCloseConfirmed')
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
|
|
@ -1368,6 +1436,61 @@ describe('SshConnection', () => {
|
|||
)
|
||||
})
|
||||
|
||||
it('composes a caller abort into system SSH relay uploads', async () => {
|
||||
vi.mocked(resolveWithSshG).mockResolvedValueOnce(createResolvedConfig())
|
||||
const conn = new SshConnection(createTarget({ configHost: 'fdpass-host' }), createCallbacks())
|
||||
const controller = new AbortController()
|
||||
let transferSignal: AbortSignal | undefined
|
||||
vi.mocked(uploadDirectoryViaSystemSsh).mockImplementationOnce(
|
||||
(_target, _localDir, _remoteDir, options) => {
|
||||
transferSignal = options?.signal
|
||||
return new Promise((_resolve, reject) => {
|
||||
transferSignal?.addEventListener('abort', () => reject(transferSignal?.reason), {
|
||||
once: true
|
||||
})
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
await conn.connect()
|
||||
const upload = conn.uploadDirectory('/tmp/local-relay', '/remote/relay', {
|
||||
signal: controller.signal
|
||||
})
|
||||
await vi.waitFor(() => expect(transferSignal).toBeDefined())
|
||||
controller.abort()
|
||||
|
||||
await expect(upload).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(transferSignal?.aborted).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps connection disconnect cancellation linked to caller-scoped relay writes', async () => {
|
||||
vi.mocked(resolveWithSshG).mockResolvedValueOnce(createResolvedConfig())
|
||||
const conn = new SshConnection(createTarget({ configHost: 'fdpass-host' }), createCallbacks())
|
||||
const controller = new AbortController()
|
||||
let transferSignal: AbortSignal | undefined
|
||||
vi.mocked(writeFileViaSystemSsh).mockImplementationOnce(
|
||||
(_target, _remotePath, _contents, options) => {
|
||||
transferSignal = options?.signal
|
||||
return new Promise((_resolve, reject) => {
|
||||
transferSignal?.addEventListener('abort', () => reject(transferSignal?.reason), {
|
||||
once: true
|
||||
})
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
await conn.connect()
|
||||
const write = conn.writeFile('/remote/relay/.version', '0.1.0', {
|
||||
signal: controller.signal
|
||||
})
|
||||
await vi.waitFor(() => expect(transferSignal).toBeDefined())
|
||||
await conn.disconnect()
|
||||
|
||||
await expect(write).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(controller.signal.aborted).toBe(false)
|
||||
expect(transferSignal?.aborted).toBe(true)
|
||||
})
|
||||
|
||||
it('keeps an upload session cancelled after the connection disconnects', async () => {
|
||||
const conn = new SshConnection(
|
||||
createTarget({ proxyCommand: 'ssh -W %h:%p bastion.example.com' }),
|
||||
|
|
|
|||
|
|
@ -41,6 +41,10 @@ import {
|
|||
import type { RemoteHostPlatform } from './ssh-remote-platform'
|
||||
import type { FileUploadSession } from '../providers/types'
|
||||
import { isSshSessionLimitError } from './ssh-session-limit-error'
|
||||
import {
|
||||
createLinkedSshFileTransferSignal,
|
||||
raceSftpFileTransferWithAbort
|
||||
} from './ssh-file-transfer-abort'
|
||||
export type { SshConnectionCallbacks } from './ssh-connection-utils'
|
||||
|
||||
type SshRemoteFileOptions = {
|
||||
|
|
@ -157,13 +161,14 @@ export class SshConnection {
|
|||
'SSH exec channel timed out',
|
||||
(callback) => client.exec(remoteCommand, callback),
|
||||
(channel) => channel.close(),
|
||||
options?.signal
|
||||
options?.signal,
|
||||
true
|
||||
),
|
||||
options?.signal
|
||||
)
|
||||
}
|
||||
|
||||
async sftp(): Promise<SFTPWrapper> {
|
||||
async sftp(signal?: AbortSignal): Promise<SFTPWrapper> {
|
||||
if (this.useSystemSshTransport) {
|
||||
throw new Error('SFTP is not available when using system SSH transport')
|
||||
}
|
||||
|
|
@ -171,12 +176,15 @@ export class SshConnection {
|
|||
throw new Error('Not connected')
|
||||
}
|
||||
const client = this.client
|
||||
return this.openSessionChannelWithRetry(() =>
|
||||
this.waitForSshCallback(
|
||||
'SSH SFTP channel timed out',
|
||||
(callback) => client.sftp(callback),
|
||||
(sftp) => sftp.end()
|
||||
)
|
||||
return this.openSessionChannelWithRetry(
|
||||
() =>
|
||||
this.waitForSshCallback(
|
||||
'SSH SFTP channel timed out',
|
||||
(callback) => client.sftp(callback),
|
||||
(sftp) => sftp.end(),
|
||||
signal
|
||||
),
|
||||
signal
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -219,10 +227,20 @@ export class SshConnection {
|
|||
timeoutMessage: string,
|
||||
register: (callback: (error: Error | undefined, value: T) => void) => void,
|
||||
cleanupLateValue?: (value: T) => void,
|
||||
signal?: AbortSignal
|
||||
signal?: AbortSignal,
|
||||
trackRemoteCommandTermination = false
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
type ChannelOpenTerminationError = Error & { sshChannelCloseConfirmed: boolean }
|
||||
let settled = false
|
||||
let unconfirmedOpenError: ChannelOpenTerminationError | null = null
|
||||
const markOpenUnconfirmed = (error: Error): Error => {
|
||||
if (!trackRemoteCommandTermination) {
|
||||
return error
|
||||
}
|
||||
unconfirmedOpenError = Object.assign(error, { sshChannelCloseConfirmed: false })
|
||||
return unconfirmedOpenError
|
||||
}
|
||||
// Why: rejecting the instant the signal aborts lets the caller proceed
|
||||
// while the in-flight channel open completes in the background and holds
|
||||
// a server-side session slot (MaxSessions). Mark the abort and settle
|
||||
|
|
@ -241,16 +259,41 @@ export class SshConnection {
|
|||
abortDeadlineTimer = setTimeout(() => {
|
||||
settled = true
|
||||
cleanup()
|
||||
reject(createSshOperationAbortError())
|
||||
reject(markOpenUnconfirmed(createSshOperationAbortError()))
|
||||
}, ABORTED_CHANNEL_CLOSE_GRACE_MS)
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
settled = true
|
||||
cleanup()
|
||||
reject(abortRequested ? createSshOperationAbortError() : new Error(timeoutMessage))
|
||||
reject(
|
||||
markOpenUnconfirmed(
|
||||
abortRequested ? createSshOperationAbortError() : new Error(timeoutMessage)
|
||||
)
|
||||
)
|
||||
}, CONNECT_TIMEOUT_MS)
|
||||
const discardLateValue = (value: T, onClose?: () => void): void => {
|
||||
const emitter = value as Partial<NodeJS.EventEmitter> & {
|
||||
resume?: () => void
|
||||
stderr?: Partial<NodeJS.EventEmitter> & { resume?: () => void }
|
||||
}
|
||||
const swallowLateError = (): void => {}
|
||||
emitter.on?.('error', swallowLateError)
|
||||
emitter.stderr?.on?.('error', swallowLateError)
|
||||
if (onClose) {
|
||||
emitter.once?.('close', onClose)
|
||||
}
|
||||
// Why: ssh2 can withhold CHANNEL_CLOSE while discarded exec streams
|
||||
// remain unread, and teardown errors have no other owner.
|
||||
emitter.resume?.()
|
||||
emitter.stderr?.resume?.()
|
||||
try {
|
||||
cleanupLateValue?.(value)
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
const rejectAfterClose = (value: T): void => {
|
||||
const abortError = createSshOperationAbortError()
|
||||
const abortError = markOpenUnconfirmed(createSshOperationAbortError())
|
||||
const emitter = value as Partial<NodeJS.EventEmitter> & {
|
||||
resume?: () => void
|
||||
stderr?: { resume?: () => void }
|
||||
|
|
@ -262,23 +305,24 @@ export class SshConnection {
|
|||
}
|
||||
finished = true
|
||||
clearTimeout(closeGraceTimer)
|
||||
emitter.removeListener?.('close', confirmAndDone)
|
||||
reject(abortError)
|
||||
}
|
||||
const confirmAndDone = (): void => {
|
||||
if (unconfirmedOpenError === abortError) {
|
||||
unconfirmedOpenError.sshChannelCloseConfirmed = true
|
||||
}
|
||||
done()
|
||||
}
|
||||
// Why: bounded — a remote that never confirms the close must not hang
|
||||
// the aborted operation forever.
|
||||
const closeGraceTimer = setTimeout(done, ABORTED_CHANNEL_CLOSE_GRACE_MS)
|
||||
if (typeof emitter.once === 'function') {
|
||||
emitter.once('close', done)
|
||||
emitter.once('close', confirmAndDone)
|
||||
}
|
||||
// Why: ssh2 withholds the 'close' event until the channel's streams
|
||||
// are drained; nobody else will ever read this discarded channel.
|
||||
emitter.resume?.()
|
||||
emitter.stderr?.resume?.()
|
||||
try {
|
||||
cleanupLateValue?.(value)
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
discardLateValue(value)
|
||||
if (typeof emitter.once !== 'function') {
|
||||
done()
|
||||
}
|
||||
|
|
@ -289,11 +333,11 @@ export class SshConnection {
|
|||
// rejected. Close that late resource so the remote channel is not
|
||||
// left open with no owner.
|
||||
if (!error && value !== undefined) {
|
||||
try {
|
||||
cleanupLateValue?.(value)
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
discardLateValue(value, () => {
|
||||
if (unconfirmedOpenError) {
|
||||
unconfirmedOpenError.sshChannelCloseConfirmed = true
|
||||
}
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
|
@ -334,23 +378,51 @@ export class SshConnection {
|
|||
async uploadDirectory(
|
||||
localDir: string,
|
||||
remoteDir: string,
|
||||
options?: SshRemoteFileOptions
|
||||
options?: SshRemoteFileOptions & { signal?: AbortSignal }
|
||||
): Promise<void> {
|
||||
if (!this.useSystemSshTransport) {
|
||||
const sftp = await this.sftp()
|
||||
try {
|
||||
const { uploadDirectory } = await import('./ssh-relay-deploy-helpers')
|
||||
await uploadDirectory(sftp, localDir, remoteDir)
|
||||
} finally {
|
||||
sftp.end()
|
||||
// Why: relay deployment timeout and connection teardown are independent;
|
||||
// either owner must stop a transfer that can otherwise outlive its lock.
|
||||
const linkedSignal = createLinkedSshFileTransferSignal(
|
||||
[this.systemOperationAbortController.signal, options?.signal].filter(
|
||||
(signal): signal is AbortSignal => signal !== undefined
|
||||
)
|
||||
)
|
||||
try {
|
||||
if (!this.useSystemSshTransport) {
|
||||
const sftp = await this.sftp(linkedSignal.signal)
|
||||
const swallowLateSftpError = (): void => {}
|
||||
let sftpEndRequested = false
|
||||
const endSftp = (): void => {
|
||||
if (!sftpEndRequested) {
|
||||
sftpEndRequested = true
|
||||
sftp.end()
|
||||
}
|
||||
}
|
||||
sftp.on('error', swallowLateSftpError)
|
||||
sftp.once('close', () => sftp.removeListener('error', swallowLateSftpError))
|
||||
try {
|
||||
const { uploadDirectory } = await import('./ssh-relay-deploy-helpers')
|
||||
await raceSftpFileTransferWithAbort(
|
||||
uploadDirectory(sftp, localDir, remoteDir),
|
||||
linkedSignal.signal,
|
||||
(onClose) => {
|
||||
sftp.once('close', onClose)
|
||||
endSftp()
|
||||
}
|
||||
)
|
||||
} finally {
|
||||
endSftp()
|
||||
}
|
||||
return
|
||||
}
|
||||
return
|
||||
await uploadDirectoryViaSystemSsh(this.target, localDir, remoteDir, {
|
||||
signal: linkedSignal.signal,
|
||||
hostPlatform: options?.hostPlatform,
|
||||
...this.getSystemSshBuildArgsOptions()
|
||||
})
|
||||
} finally {
|
||||
linkedSignal.dispose()
|
||||
}
|
||||
await uploadDirectoryViaSystemSsh(this.target, localDir, remoteDir, {
|
||||
signal: this.systemOperationAbortController.signal,
|
||||
hostPlatform: options?.hostPlatform,
|
||||
...this.getSystemSshBuildArgsOptions()
|
||||
})
|
||||
}
|
||||
|
||||
async downloadFile(
|
||||
|
|
@ -404,55 +476,74 @@ export class SshConnection {
|
|||
async writeFile(
|
||||
remotePath: string,
|
||||
contents: string,
|
||||
options?: SshRemoteFileOptions
|
||||
options?: SshRemoteFileOptions & { signal?: AbortSignal }
|
||||
): Promise<void> {
|
||||
if (!this.useSystemSshTransport) {
|
||||
const sftp = await this.sftp()
|
||||
const swallowLateSftpError = (): void => {}
|
||||
sftp.on('error', swallowLateSftpError)
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ws = sftp.createWriteStream(remotePath)
|
||||
let settled = false
|
||||
const cleanup = (): void => {
|
||||
ws.removeListener('close', onClose)
|
||||
ws.removeListener('error', onError)
|
||||
// Keep package/version writes under the same dual cancellation contract as uploads.
|
||||
const linkedSignal = createLinkedSshFileTransferSignal(
|
||||
[this.systemOperationAbortController.signal, options?.signal].filter(
|
||||
(signal): signal is AbortSignal => signal !== undefined
|
||||
)
|
||||
)
|
||||
try {
|
||||
if (!this.useSystemSshTransport) {
|
||||
const sftp = await this.sftp(linkedSignal.signal)
|
||||
const swallowLateSftpError = (): void => {}
|
||||
let sftpEndRequested = false
|
||||
const endSftp = (): void => {
|
||||
if (!sftpEndRequested) {
|
||||
sftpEndRequested = true
|
||||
sftp.end()
|
||||
}
|
||||
const onClose = (): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
sftp.on('error', swallowLateSftpError)
|
||||
sftp.once('close', () => sftp.removeListener('error', swallowLateSftpError))
|
||||
try {
|
||||
const write = new Promise<void>((resolve, reject) => {
|
||||
const ws = sftp.createWriteStream(remotePath)
|
||||
let settled = false
|
||||
const cleanup = (): void => {
|
||||
sftp.removeListener('error', onError)
|
||||
ws.removeListener('close', onClose)
|
||||
ws.removeListener('error', onError)
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
const onError = (err: Error): void => {
|
||||
sftp.removeListener('error', onError)
|
||||
if (settled) {
|
||||
return
|
||||
const onClose = (): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
resolve()
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
reject(err)
|
||||
}
|
||||
sftp.prependOnceListener('error', onError)
|
||||
ws.once('close', onClose)
|
||||
ws.once('error', onError)
|
||||
ws.end(contents)
|
||||
})
|
||||
} finally {
|
||||
sftp.end()
|
||||
setImmediate(() => {
|
||||
sftp.removeListener('error', swallowLateSftpError)
|
||||
})
|
||||
const onError = (err: Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
reject(err)
|
||||
}
|
||||
sftp.prependOnceListener('error', onError)
|
||||
ws.once('close', onClose)
|
||||
ws.once('error', onError)
|
||||
ws.end(contents)
|
||||
})
|
||||
await raceSftpFileTransferWithAbort(write, linkedSignal.signal, (onClose) => {
|
||||
sftp.once('close', onClose)
|
||||
endSftp()
|
||||
})
|
||||
} finally {
|
||||
endSftp()
|
||||
}
|
||||
return
|
||||
}
|
||||
return
|
||||
await writeFileViaSystemSsh(this.target, remotePath, contents, {
|
||||
signal: linkedSignal.signal,
|
||||
hostPlatform: options?.hostPlatform,
|
||||
...this.getSystemSshBuildArgsOptions()
|
||||
})
|
||||
} finally {
|
||||
linkedSignal.dispose()
|
||||
}
|
||||
await writeFileViaSystemSsh(this.target, remotePath, contents, {
|
||||
signal: this.systemOperationAbortController.signal,
|
||||
hostPlatform: options?.hostPlatform,
|
||||
...this.getSystemSshBuildArgsOptions()
|
||||
})
|
||||
}
|
||||
|
||||
async writeBuffer(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,55 @@
|
|||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { raceSftpFileTransferWithAbort } from './ssh-file-transfer-abort'
|
||||
|
||||
describe('raceSftpFileTransferWithAbort', () => {
|
||||
it('waits for confirmed SFTP close before rejecting an abort', async () => {
|
||||
const controller = new AbortController()
|
||||
let confirmClose: () => void = () => {}
|
||||
const promise = raceSftpFileTransferWithAbort(
|
||||
new Promise<void>(() => {}),
|
||||
controller.signal,
|
||||
(onClose) => {
|
||||
confirmClose = onClose
|
||||
}
|
||||
)
|
||||
|
||||
controller.abort()
|
||||
const pending = await Promise.race([
|
||||
promise.then(
|
||||
() => 'settled',
|
||||
() => 'settled'
|
||||
),
|
||||
Promise.resolve('pending')
|
||||
])
|
||||
expect(pending).toBe('pending')
|
||||
|
||||
confirmClose()
|
||||
await expect(promise).rejects.toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: true
|
||||
})
|
||||
})
|
||||
|
||||
it('marks teardown unconfirmed when SFTP never closes', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const controller = new AbortController()
|
||||
const promise = raceSftpFileTransferWithAbort(
|
||||
new Promise<void>(() => {}),
|
||||
controller.signal,
|
||||
() => {}
|
||||
)
|
||||
const outcome = promise.catch((error: Error) => error)
|
||||
|
||||
controller.abort()
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
await expect(outcome).resolves.toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
import { createSshOperationAbortError } from './ssh-connection-utils'
|
||||
|
||||
export function createLinkedSshFileTransferSignal(signals: readonly AbortSignal[]): {
|
||||
signal: AbortSignal
|
||||
dispose: () => void
|
||||
} {
|
||||
const controller = new AbortController()
|
||||
const listeners = signals.map((signal) => {
|
||||
const listener = (): void => controller.abort(signal.reason)
|
||||
if (signal.aborted) {
|
||||
listener()
|
||||
} else {
|
||||
signal.addEventListener('abort', listener, { once: true })
|
||||
}
|
||||
return { signal, listener }
|
||||
})
|
||||
return {
|
||||
signal: controller.signal,
|
||||
dispose: () => {
|
||||
for (const { signal, listener } of listeners) {
|
||||
signal.removeEventListener('abort', listener)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function raceSftpFileTransferWithAbort<T>(
|
||||
operation: Promise<T>,
|
||||
signal: AbortSignal,
|
||||
closeSftp: (onClose: () => void) => void
|
||||
): Promise<T> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false
|
||||
let abortError: (Error & { sshChannelCloseConfirmed: boolean }) | null = null
|
||||
let closeGraceTimer: ReturnType<typeof setTimeout> | null = null
|
||||
const settle = (fn: typeof resolve | typeof reject, value: T | Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
if (closeGraceTimer) {
|
||||
clearTimeout(closeGraceTimer)
|
||||
}
|
||||
signal.removeEventListener('abort', onAbort)
|
||||
fn(value as never)
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
// Why: rejecting the caller is insufficient; ending SFTP stops the
|
||||
// abandoned transfer from mutating a successor relay install.
|
||||
abortError = Object.assign(createSshOperationAbortError(), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
closeGraceTimer = setTimeout(() => settle(reject, abortError!), 5_000)
|
||||
closeSftp(() => {
|
||||
abortError!.sshChannelCloseConfirmed = true
|
||||
settle(reject, abortError!)
|
||||
})
|
||||
}
|
||||
signal.addEventListener('abort', onAbort, { once: true })
|
||||
void operation.then(
|
||||
(value) => {
|
||||
if (!abortError) {
|
||||
settle(resolve, value)
|
||||
}
|
||||
},
|
||||
(error: unknown) => {
|
||||
if (!abortError) {
|
||||
settle(reject, error instanceof Error ? error : new Error(String(error)))
|
||||
}
|
||||
}
|
||||
)
|
||||
if (signal.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -141,15 +141,20 @@ export function formatMissingToolchainError(
|
|||
// callers fall back to the original install error in those cases.
|
||||
export async function probeBuildToolchain(
|
||||
conn: SshConnection,
|
||||
hostPlatform: RemoteHostPlatform
|
||||
hostPlatform: RemoteHostPlatform,
|
||||
signal?: AbortSignal
|
||||
): Promise<BuildToolchainStatus | null> {
|
||||
if (isWindowsRemoteHost(hostPlatform)) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const output = await execCommand(conn, buildToolchainProbeCommand(), { wrapCommand: true })
|
||||
const output = await execCommand(conn, buildToolchainProbeCommand(), {
|
||||
wrapCommand: true,
|
||||
signal
|
||||
})
|
||||
return parseBuildToolchainProbe(output)
|
||||
} catch {
|
||||
signal?.throwIfAborted()
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ vi.mock('./ssh-relay-deploy-helpers', () => ({
|
|||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
error instanceof Error &&
|
||||
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false,
|
||||
execCommand: vi.fn()
|
||||
}))
|
||||
|
||||
|
|
@ -101,8 +104,10 @@ describe('cross-version isolation', () => {
|
|||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // tagged POSIX platform probe
|
||||
'/home/u', // echo $HOME
|
||||
'MISSING', // isRelayAlreadyInstalled (v2 dir doesn't exist)
|
||||
'OPEN', // no sibling GC claim
|
||||
'', // mkdir -p remoteRelayDir (v2)
|
||||
'OK', // mkdir lock OK
|
||||
'OPEN', // GC did not claim while the install lock was acquired
|
||||
'MISSING', // re-probe after lock → still missing → proceed with install
|
||||
'', // mkdir remoteDir (uploadRelay)
|
||||
'', // chmod +x node
|
||||
|
|
@ -111,9 +116,9 @@ describe('cross-version isolation', () => {
|
|||
'ORCA-NPTY-PROBE-OK\n', // node -e require() load-test (post-install verify)
|
||||
'', // rm -f probe-stderr (best-effort cleanup after probe resolved)
|
||||
'', // touch .install-complete (finalizeInstall)
|
||||
'', // rm -rf .install-lock
|
||||
'DEAD', // launch socket probe
|
||||
'READY', // socket poll
|
||||
'', // release .install-lock after relay liveness is observable
|
||||
// GC scan begins here
|
||||
'relay-0.1.0+v1hash\nrelay-0.1.0+v2hash\n', // ls listing
|
||||
'OPEN', // v1 lock probe (siblings only — current dir is v2)
|
||||
|
|
|
|||
|
|
@ -11,9 +11,10 @@ import {
|
|||
|
||||
function createMockChannel(): ClientChannel {
|
||||
return Object.assign(new EventEmitter(), {
|
||||
stderr: new EventEmitter(),
|
||||
stderr: Object.assign(new EventEmitter(), { resume: vi.fn() }),
|
||||
stdin: { write: vi.fn() },
|
||||
close: vi.fn()
|
||||
close: vi.fn(),
|
||||
resume: vi.fn()
|
||||
}) as unknown as ClientChannel
|
||||
}
|
||||
|
||||
|
|
@ -29,6 +30,44 @@ async function execCommandRejection(promise: Promise<string>): Promise<Error> {
|
|||
}
|
||||
|
||||
describe('waitForSentinel', () => {
|
||||
it('closes and rejects with AbortError while clearing startup resources on abort', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const channel = createMockChannel()
|
||||
const controller = new AbortController()
|
||||
const removeListener = vi.spyOn(controller.signal, 'removeEventListener')
|
||||
const transportPromise = waitForSentinel(channel, controller.signal)
|
||||
|
||||
expect(vi.getTimerCount()).toBe(1)
|
||||
controller.abort()
|
||||
|
||||
await expect(transportPromise).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(channel.close).toHaveBeenCalledTimes(1)
|
||||
expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function))
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('removes the abort listener and timers when startup fails before the sentinel', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const channel = createMockChannel()
|
||||
const controller = new AbortController()
|
||||
const removeListener = vi.spyOn(controller.signal, 'removeEventListener')
|
||||
const transportPromise = waitForSentinel(channel, controller.signal)
|
||||
|
||||
channel.emit('error', new Error('remote host rebooted'))
|
||||
|
||||
await expect(transportPromise).rejects.toThrow('remote host rebooted')
|
||||
expect(removeListener).toHaveBeenCalledWith('abort', expect.any(Function))
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('buffers post-sentinel chunks until the transport subscribes', async () => {
|
||||
const channel = createMockChannel()
|
||||
const transportPromise = waitForSentinel(channel)
|
||||
|
|
@ -168,6 +207,32 @@ describe('waitForSentinel', () => {
|
|||
})
|
||||
|
||||
describe('execCommand', () => {
|
||||
it('waits for channel close before rejecting a timed-out remote command', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const channel = createMockChannel()
|
||||
const conn = { exec: vi.fn().mockResolvedValue(channel) }
|
||||
const commandPromise = execCommand(conn as never, 'npm rebuild', { timeoutMs: 1_000 })
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(channel.close).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
await Promise.race([
|
||||
commandPromise.then(
|
||||
() => 'settled',
|
||||
() => 'settled'
|
||||
),
|
||||
Promise.resolve('pending')
|
||||
])
|
||||
).toBe('pending')
|
||||
|
||||
channel.emit('close', 0)
|
||||
await expect(commandPromise).rejects.toThrow('timed out after 1s')
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects on command channel errors instead of emitting uncaught errors', async () => {
|
||||
const channel = createMockChannel()
|
||||
const conn = {
|
||||
|
|
@ -177,6 +242,8 @@ describe('execCommand', () => {
|
|||
|
||||
await Promise.resolve()
|
||||
expect(() => channel.emit('error', new Error('remote host rebooted'))).not.toThrow()
|
||||
expect(channel.close).toHaveBeenCalledOnce()
|
||||
channel.emit('close', 1)
|
||||
await expect(commandPromise).rejects.toThrow('remote host rebooted')
|
||||
expect(channel.listenerCount('error')).toBe(0)
|
||||
expect(channel.listenerCount('data')).toBe(0)
|
||||
|
|
@ -224,6 +291,22 @@ describe('execCommand', () => {
|
|||
expect(error.message).toContain('gnu++20')
|
||||
})
|
||||
|
||||
it('bounds command output while preserving the actionable tail', async () => {
|
||||
const channel = createMockChannel()
|
||||
const conn = { exec: vi.fn().mockResolvedValue(channel) }
|
||||
const commandPromise = execCommand(conn as never, 'npm rebuild')
|
||||
|
||||
await Promise.resolve()
|
||||
channel.emit('data', Buffer.from(`old-prefix-${'x'.repeat(1024 * 1024)}`))
|
||||
channel.emit('data', Buffer.from('gyp ERR! tail diagnosis'))
|
||||
channel.emit('close', 1)
|
||||
|
||||
const error = await execCommandRejection(commandPromise)
|
||||
expect(error.message).not.toContain('old-prefix')
|
||||
expect(error.message).toContain('gyp ERR! tail diagnosis')
|
||||
expect(error.message.length).toBeLessThan(1024 * 1024 + 200)
|
||||
})
|
||||
|
||||
it('keeps the merged error message greppable by the build-toolchain probe', async () => {
|
||||
const channel = createMockChannel()
|
||||
const conn = {
|
||||
|
|
@ -260,14 +343,26 @@ describe('execCommand', () => {
|
|||
await Promise.resolve()
|
||||
const rejection = expect(commandPromise).rejects.toThrow('timed out')
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(channel.close).toHaveBeenCalledOnce()
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
await rejection
|
||||
expect(channel.close).toHaveBeenCalledOnce()
|
||||
expect(channel.listenerCount('error')).toBe(0)
|
||||
expect(channel.resume).toHaveBeenCalledOnce()
|
||||
expect(channel.stderr.resume).toHaveBeenCalledOnce()
|
||||
expect(() => channel.emit('error', new Error('late channel error'))).not.toThrow()
|
||||
expect(() => channel.stderr.emit('error', new Error('late stderr error'))).not.toThrow()
|
||||
channel.emit('data', Buffer.from('late stdout'))
|
||||
channel.stderr.emit('data', Buffer.from('late stderr'))
|
||||
expect(channel.listenerCount('error')).toBe(1)
|
||||
expect(channel.listenerCount('data')).toBe(0)
|
||||
expect(channel.listenerCount('close')).toBe(1)
|
||||
expect(channel.stderr.listenerCount('error')).toBe(1)
|
||||
expect(channel.stderr.listenerCount('data')).toBe(0)
|
||||
channel.emit('close', 0)
|
||||
expect(channel.listenerCount('error')).toBe(0)
|
||||
expect(channel.listenerCount('close')).toBe(0)
|
||||
expect(channel.stderr.listenerCount('error')).toBe(0)
|
||||
expect(channel.stderr.listenerCount('data')).toBe(0)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
|
|
@ -309,6 +404,48 @@ describe('execCommand', () => {
|
|||
expect(channel.stderr.listenerCount('data')).toBe(0)
|
||||
})
|
||||
|
||||
it('keeps system-SSH command termination unconfirmed after the local child closes', async () => {
|
||||
const channel = createMockChannel()
|
||||
const controller = new AbortController()
|
||||
const conn = {
|
||||
exec: vi.fn().mockResolvedValue(channel),
|
||||
usesSystemSshTransport: vi.fn().mockReturnValueOnce(true).mockReturnValue(false)
|
||||
}
|
||||
const commandPromise = execCommand(conn as never, 'npm install', {
|
||||
signal: controller.signal
|
||||
})
|
||||
|
||||
await Promise.resolve()
|
||||
controller.abort()
|
||||
channel.emit('close', 0)
|
||||
|
||||
await expect(commandPromise).rejects.toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
})
|
||||
|
||||
it('tags an externally closed system-SSH command as unconfirmed', async () => {
|
||||
const channel = createMockChannel() as ClientChannel & { _closeRequested?: boolean }
|
||||
channel.close = vi.fn(() => {
|
||||
channel._closeRequested = true
|
||||
})
|
||||
const conn = {
|
||||
exec: vi.fn().mockResolvedValue(channel),
|
||||
usesSystemSshTransport: vi.fn().mockReturnValue(true)
|
||||
}
|
||||
const commandPromise = execCommand(conn as never, 'npm install')
|
||||
|
||||
await Promise.resolve()
|
||||
channel.close()
|
||||
channel.emit('close', 1)
|
||||
|
||||
await expect(commandPromise).rejects.toMatchObject({
|
||||
name: 'AbortError',
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
})
|
||||
|
||||
it('handles aborts that happen while the SSH exec channel is still opening', async () => {
|
||||
const channel = createMockChannel()
|
||||
const controller = new AbortController()
|
||||
|
|
@ -357,6 +494,8 @@ describe('execCommand', () => {
|
|||
expect(conn.exec).toHaveBeenCalledWith('npm install', { wrapCommand: false })
|
||||
const rejection = expect(commandPromise).rejects.toThrow('timed out after 240s')
|
||||
await vi.advanceTimersByTimeAsync(240_000)
|
||||
expect(channel.close).toHaveBeenCalledOnce()
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
await rejection
|
||||
expect(channel.close).toHaveBeenCalledOnce()
|
||||
|
|
|
|||
|
|
@ -1,18 +1,21 @@
|
|||
import type { ClientChannel } from 'ssh2'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { createSshOperationAbortError, type SshExecOptions } from './ssh-connection-utils'
|
||||
import { createSshOperationAbortError } from './ssh-connection-utils'
|
||||
import { RELAY_SENTINEL, RELAY_SENTINEL_TIMEOUT_MS } from './relay-protocol'
|
||||
import type { MultiplexerTransport } from './ssh-channel-multiplexer'
|
||||
import { buildRelayVersionMismatchError } from './ssh-relay-handshake-mismatch'
|
||||
|
||||
export { uploadFile, uploadDirectory, mkdirSftp } from './sftp-upload'
|
||||
export { execCommand, isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
|
||||
|
||||
// ── Sentinel detection ────────────────────────────────────────────────
|
||||
|
||||
const MAX_RELAY_STARTUP_BUFFER_BYTES = 64 * 1024
|
||||
const RELAY_SENTINEL_BUFFER = Buffer.from(RELAY_SENTINEL, 'utf-8')
|
||||
|
||||
export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTransport> {
|
||||
export function waitForSentinel(
|
||||
channel: ClientChannel,
|
||||
signal?: AbortSignal
|
||||
): Promise<MultiplexerTransport> {
|
||||
return new Promise<MultiplexerTransport>((resolve, reject) => {
|
||||
let sentinelReceived = false
|
||||
let settled = false
|
||||
|
|
@ -39,17 +42,14 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
|
||||
const timeout = setTimeout(() => {
|
||||
timeoutFired = true
|
||||
channel.close()
|
||||
timeoutGraceTimer = setTimeout(() => {
|
||||
if (!settled) {
|
||||
settled = true
|
||||
reject(
|
||||
new Error(
|
||||
`Relay failed to start within ${RELAY_SENTINEL_TIMEOUT_MS / 1000}s.${stderrOutput ? ` stderr: ${stderrOutput.trim()}` : ''}`
|
||||
)
|
||||
rejectStartup(
|
||||
new Error(
|
||||
`Relay failed to start within ${RELAY_SENTINEL_TIMEOUT_MS / 1000}s.${stderrOutput ? ` stderr: ${stderrOutput.trim()}` : ''}`
|
||||
)
|
||||
}
|
||||
)
|
||||
}, TIMEOUT_GRACE_MS)
|
||||
channel.close()
|
||||
}, RELAY_SENTINEL_TIMEOUT_MS)
|
||||
|
||||
const cancelTimers = (): void => {
|
||||
|
|
@ -59,6 +59,31 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
timeoutGraceTimer = null
|
||||
}
|
||||
}
|
||||
const cleanupStartup = (): void => {
|
||||
cancelTimers()
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
}
|
||||
const rejectStartup = (err: Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanupStartup()
|
||||
reject(err)
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanupStartup()
|
||||
channel.close()
|
||||
reject(createSshOperationAbortError())
|
||||
}
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
|
||||
channel.on('exit', (code: number | null) => {
|
||||
if (typeof code === 'number') {
|
||||
|
|
@ -87,12 +112,8 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
}
|
||||
|
||||
const failOrClose = (err: Error): void => {
|
||||
cancelTimers()
|
||||
if (!sentinelReceived) {
|
||||
if (!settled) {
|
||||
settled = true
|
||||
reject(err)
|
||||
}
|
||||
rejectStartup(err)
|
||||
return
|
||||
}
|
||||
notifyClosed()
|
||||
|
|
@ -113,9 +134,7 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
|
||||
channel.on('close', () => {
|
||||
if (!sentinelReceived) {
|
||||
cancelTimers()
|
||||
if (!settled) {
|
||||
settled = true
|
||||
// Why: a wire-handshake mismatch on the daemon side closes the
|
||||
// socket; --connect prints the mismatch detail to stderr and exits
|
||||
// with code 42 BEFORE writing the sentinel. Translate that into a
|
||||
|
|
@ -126,13 +145,13 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
// grace window so the close handler can deliver the exit code.
|
||||
const versionMismatchError = buildRelayVersionMismatchError(lastExitCode, stderrOutput)
|
||||
if (versionMismatchError) {
|
||||
reject(versionMismatchError)
|
||||
rejectStartup(versionMismatchError)
|
||||
return
|
||||
}
|
||||
const timeoutSuffix = timeoutFired
|
||||
? ` (after ${RELAY_SENTINEL_TIMEOUT_MS / 1000}s sentinel timeout)`
|
||||
: ''
|
||||
reject(
|
||||
rejectStartup(
|
||||
new Error(
|
||||
`Relay process exited before ready${timeoutSuffix}.${stderrOutput ? ` stderr: ${stderrOutput.trim()}` : ''}`
|
||||
)
|
||||
|
|
@ -184,7 +203,8 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
|
||||
if (sentinelIdx !== -1) {
|
||||
sentinelReceived = true
|
||||
cancelTimers()
|
||||
settled = true
|
||||
cleanupStartup()
|
||||
|
||||
const afterSentinelOffset =
|
||||
sentinelIdx + RELAY_SENTINEL_BUFFER.length - bufferedStdout.length
|
||||
|
|
@ -193,8 +213,6 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
if (afterSentinel.length > 0) {
|
||||
pendingAfterSentinel = afterSentinel
|
||||
}
|
||||
settled = true
|
||||
|
||||
const transport: MultiplexerTransport = {
|
||||
write: (buf: Buffer) => channel.stdin.write(buf),
|
||||
onData: (cb) => {
|
||||
|
|
@ -233,97 +251,3 @@ export function waitForSentinel(channel: ClientChannel): Promise<MultiplexerTran
|
|||
})
|
||||
})
|
||||
}
|
||||
|
||||
// ── Remote command execution ──────────────────────────────────────────
|
||||
|
||||
const EXEC_TIMEOUT_MS = 30_000
|
||||
type ExecCommandOptions = SshExecOptions & {
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
export async function execCommand(
|
||||
conn: SshConnection,
|
||||
command: string,
|
||||
options?: ExecCommandOptions
|
||||
): Promise<string> {
|
||||
const { timeoutMs = EXEC_TIMEOUT_MS, ...execOptions } = options ?? {}
|
||||
const signal = options?.signal
|
||||
if (signal?.aborted) {
|
||||
throw createSshOperationAbortError()
|
||||
}
|
||||
const channel = await conn.exec(command, execOptions)
|
||||
return new Promise((resolve, reject) => {
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let settled = false
|
||||
|
||||
const cleanup = (): void => {
|
||||
clearTimeout(timeout)
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
channel.off('error', fail)
|
||||
channel.stderr.off('error', fail)
|
||||
channel.off('data', onStdoutData)
|
||||
channel.stderr.off('data', onStderrData)
|
||||
channel.off('close', onClose)
|
||||
}
|
||||
const settle = (fn: typeof resolve | typeof reject, val: string | Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
fn(val as never)
|
||||
}
|
||||
const fail = (err: Error): void => {
|
||||
settle(reject, abortRequested ? createSshOperationAbortError() : err)
|
||||
}
|
||||
// Why: sshd counts the session against MaxSessions until CHANNEL_CLOSE
|
||||
// completes. Settling on abort before the channel actually closes lets the
|
||||
// concurrent-bootstrap sequential fallback reissue an exec while the slot
|
||||
// is still held, so it gets refused again. Close and settle from onClose.
|
||||
let abortRequested = false
|
||||
const onAbort = (): void => {
|
||||
abortRequested = true
|
||||
channel.close()
|
||||
}
|
||||
const onStdoutData = (data: Buffer): void => {
|
||||
stdout += data.toString('utf-8')
|
||||
}
|
||||
const onStderrData = (data: Buffer): void => {
|
||||
stderr += data.toString('utf-8')
|
||||
}
|
||||
const onClose = (code: number): void => {
|
||||
if (abortRequested) {
|
||||
settle(reject, createSshOperationAbortError())
|
||||
} else if (code !== 0) {
|
||||
// Why: on the system-ssh transport channel.stderr carries local OpenSSH
|
||||
// client noise; preferring it masks the real failure in stdout (2>&1).
|
||||
const output = [stderr.trim(), stdout.trim()].filter(Boolean).join('\n')
|
||||
settle(reject, new Error(`Command "${command}" failed (exit ${code}): ${output}`))
|
||||
} else {
|
||||
settle(resolve, stdout)
|
||||
}
|
||||
}
|
||||
const timeout = setTimeout(() => {
|
||||
channel.close()
|
||||
settle(
|
||||
reject,
|
||||
abortRequested
|
||||
? createSshOperationAbortError()
|
||||
: new Error(`Command "${command}" timed out after ${timeoutMs / 1000}s`)
|
||||
)
|
||||
}, timeoutMs)
|
||||
|
||||
// Why: remote reboot tears down exec channels with stream errors. Without
|
||||
// scoped listeners, Node treats those as uncaught exceptions.
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
channel.on('error', fail)
|
||||
channel.stderr.on('error', fail)
|
||||
channel.on('data', onStdoutData)
|
||||
channel.stderr.on('data', onStderrData)
|
||||
channel.on('close', onClose)
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
// npm install on a cold Windows cache plus antivirus scanning can exceed the
|
||||
// default 30s exec timeout.
|
||||
export const NATIVE_DEPS_COMMAND_TIMEOUT_MS = 240_000
|
||||
|
||||
// Why: a missing binding can require both install and rebuild while the same
|
||||
// install lock is held. Concurrent first installs must wait for that valid
|
||||
// holder instead of failing halfway through its bounded work.
|
||||
const NATIVE_DEPS_REPAIR_BUDGET_MS = 2 * NATIVE_DEPS_COMMAND_TIMEOUT_MS
|
||||
|
||||
// Why: native repair also runs separately bounded chmod/probe/diagnostic/
|
||||
// finalize commands around install + rebuild. Keep the outer bound above that
|
||||
// valid worst path while remaining below the 20-minute stale-lock threshold.
|
||||
export const RELAY_DEPLOY_TIMEOUT_MS = NATIVE_DEPS_REPAIR_BUDGET_MS + 7 * 60_000
|
||||
|
|
@ -42,6 +42,9 @@ vi.mock('./ssh-relay-deploy-helpers', () => ({
|
|||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
error instanceof Error &&
|
||||
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false,
|
||||
execCommand: vi.fn().mockResolvedValue('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
}))
|
||||
|
||||
|
|
@ -49,27 +52,38 @@ vi.mock('./ssh-remote-node-resolution', () => ({
|
|||
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
|
||||
}))
|
||||
|
||||
// Why: the versioned-install module shells out to the remote for install
|
||||
// state, lock acquisition, and GC. Tests stub these to no-ops so the deploy
|
||||
// happy-path is exercised without a real SSH connection.
|
||||
// Why: the versioned-install modules shell out for install state, locking,
|
||||
// and GC. Stub them so deploy tests need no real SSH connection.
|
||||
vi.mock('./ssh-relay-versioned-install', () => ({
|
||||
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+abcdef012345'),
|
||||
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
|
||||
isRelayAlreadyInstalled: vi.fn().mockResolvedValue(true),
|
||||
acquireInstallLock: vi.fn().mockResolvedValue(undefined),
|
||||
finalizeInstall: vi.fn().mockResolvedValue(undefined),
|
||||
abandonInstall: vi.fn().mockResolvedValue(undefined),
|
||||
gcOldRelayVersions: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-install-lock', () => ({
|
||||
acquireInstallLock: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-repair-lock', () => ({
|
||||
tryAcquireRelayRepairLock: vi.fn().mockResolvedValue('acquired')
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-connection-utils', () => ({
|
||||
shellEscape: (s: string) => `'${s}'`
|
||||
shellEscape: (s: string) => `'${s}'`,
|
||||
createSshOperationAbortError: () =>
|
||||
Object.assign(new Error('SSH operation was cancelled'), {
|
||||
name: 'AbortError'
|
||||
})
|
||||
}))
|
||||
|
||||
import { deployAndLaunchRelay } from './ssh-relay-deploy'
|
||||
import { execCommand, waitForSentinel } from './ssh-relay-deploy-helpers'
|
||||
import { resolveRemoteNodePath } from './ssh-remote-node-resolution'
|
||||
import { isRelayAlreadyInstalled } from './ssh-relay-versioned-install'
|
||||
import { acquireInstallLock } from './ssh-relay-install-lock'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import type * as SshRemoteNodeResolution from './ssh-remote-node-resolution'
|
||||
import {
|
||||
|
|
@ -133,7 +147,8 @@ describe('deployAndLaunchRelay', () => {
|
|||
|
||||
expect(mockExecCommand).toHaveBeenCalledWith(
|
||||
conn,
|
||||
"printf '\\n%s ' '__ORCA_REMOTE_PLATFORM__'; uname -sm"
|
||||
"printf '\\n%s ' '__ORCA_REMOTE_PLATFORM__'; uname -sm",
|
||||
expect.objectContaining({ signal: expect.any(AbortSignal) })
|
||||
)
|
||||
})
|
||||
|
||||
|
|
@ -153,6 +168,26 @@ describe('deployAndLaunchRelay', () => {
|
|||
expect(progress).toContain('Starting relay...')
|
||||
})
|
||||
|
||||
it('does not launch fresh after unconfirmed stale-socket cleanup', async () => {
|
||||
const conn = makeMockConnection()
|
||||
const unconfirmedCleanup = Object.assign(new Error('socket cleanup still running'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('stale relay reconnect failed'))
|
||||
vi.mocked(execCommand)
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/user')
|
||||
.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK')
|
||||
.mockResolvedValueOnce('ALIVE')
|
||||
.mockRejectedValueOnce(unconfirmedCleanup)
|
||||
|
||||
await expect(deployAndLaunchRelay(conn)).rejects.toBe(unconfirmedCleanup)
|
||||
|
||||
const commands = vi.mocked(conn.exec).mock.calls.map(([command]) => command)
|
||||
expect(commands).toHaveLength(1)
|
||||
expect(commands.some((command) => command.includes('--detached'))).toBe(false)
|
||||
})
|
||||
|
||||
it('resolves the remote node path once per deploy', async () => {
|
||||
const conn = makeMockConnection()
|
||||
const mockExecCommand = vi.mocked(execCommand)
|
||||
|
|
@ -202,7 +237,7 @@ describe('deployAndLaunchRelay', () => {
|
|||
assertionError = err
|
||||
} finally {
|
||||
// Drain the rest of the happy path so a failed assertion does not leave
|
||||
// the deploy promise pending until its 300s timeout.
|
||||
// the deploy promise pending until the overall deploy timeout.
|
||||
mockExecCommand.mockResolvedValueOnce('ORCA-NATIVE-DEPS-OK') // native deps probe
|
||||
mockExecCommand.mockResolvedValueOnce('DEAD') // socket probe
|
||||
mockExecCommand.mockResolvedValueOnce('READY') // socket poll
|
||||
|
|
@ -287,7 +322,10 @@ describe('deployAndLaunchRelay', () => {
|
|||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(2)
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(3)
|
||||
expect(vi.mocked(isRelayAlreadyInstalled).mock.calls[2]?.[3]).toMatchObject({
|
||||
rethrowSessionLimitErrors: true
|
||||
})
|
||||
expect(resolveRemoteNodePath).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
|
|
@ -314,11 +352,17 @@ describe('deployAndLaunchRelay', () => {
|
|||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(2)
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(3)
|
||||
expect(vi.mocked(isRelayAlreadyInstalled).mock.calls[0]?.[3]).toMatchObject({
|
||||
rethrowSessionLimitErrors: true
|
||||
})
|
||||
expect(vi.mocked(isRelayAlreadyInstalled).mock.calls[1]?.[3]).toBeUndefined()
|
||||
expect(vi.mocked(isRelayAlreadyInstalled).mock.calls[1]?.[3]).toMatchObject({
|
||||
rethrowSessionLimitErrors: undefined,
|
||||
signal: expect.any(AbortSignal)
|
||||
})
|
||||
expect(vi.mocked(isRelayAlreadyInstalled).mock.calls[2]?.[3]).toMatchObject({
|
||||
rethrowSessionLimitErrors: true
|
||||
})
|
||||
expect(resolveRemoteNodePath).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
|
|
@ -499,7 +543,10 @@ describe('deployAndLaunchRelay', () => {
|
|||
expect(sawLegacyDir).toBe(false)
|
||||
})
|
||||
|
||||
it('has a 300-second overall timeout', async () => {
|
||||
it('bounds the overall deploy so install + rebuild both fit under the timeout', async () => {
|
||||
// Why: the outer bound must exceed the worst-case sequential native-deps
|
||||
// work — a first install (240s) AND a follow-up rebuild (240s) — so a
|
||||
// legitimate install-then-rebuild is not falsely timed out mid-repair.
|
||||
const conn = makeMockConnection()
|
||||
const mockExecCommand = vi.mocked(execCommand)
|
||||
|
||||
|
|
@ -511,15 +558,140 @@ describe('deployAndLaunchRelay', () => {
|
|||
// Catch the rejection immediately to avoid unhandled rejection warning
|
||||
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
|
||||
// Not timed out yet at the old 300s bound (install + rebuild need more).
|
||||
await vi.advanceTimersByTimeAsync(301_000)
|
||||
expect(await Promise.race([promise, Promise.resolve('pending')])).toBe('pending')
|
||||
|
||||
await vi.advanceTimersByTimeAsync(600_000)
|
||||
|
||||
const result = await promise
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toBe('Relay deployment timed out after 300s')
|
||||
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
|
||||
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('aborts a contended install-lock wait at the overall deploy timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const conn = makeMockConnection()
|
||||
vi.mocked(execCommand)
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/user')
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValueOnce(false)
|
||||
let lockSignal: AbortSignal | undefined
|
||||
vi.mocked(acquireInstallLock).mockImplementationOnce((_conn, _dir, _host, options) => {
|
||||
lockSignal = options?.signal
|
||||
return new Promise<void>((_resolve, reject) => {
|
||||
lockSignal?.addEventListener('abort', () => reject(lockSignal?.reason), { once: true })
|
||||
})
|
||||
})
|
||||
|
||||
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(acquireInstallLock).toHaveBeenCalledTimes(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(900_000)
|
||||
|
||||
const result = await promise
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
|
||||
expect(lockSignal?.aborted).toBe(true)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('aborts an in-progress relay upload at the overall deploy timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const conn = makeMockConnection()
|
||||
vi.mocked(execCommand)
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/user')
|
||||
.mockResolvedValueOnce('') // mkdir remote relay dir
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValueOnce(false).mockResolvedValueOnce(false)
|
||||
let uploadSignal: AbortSignal | undefined
|
||||
conn.uploadDirectory = vi.fn((_localDir, _remoteDir, options) => {
|
||||
uploadSignal = options?.signal
|
||||
return new Promise<void>((_resolve, reject) => {
|
||||
uploadSignal?.addEventListener('abort', () => reject(uploadSignal?.reason), {
|
||||
once: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(conn.uploadDirectory).toHaveBeenCalledTimes(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(900_000)
|
||||
|
||||
const result = await promise
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
|
||||
expect(uploadSignal?.aborted).toBe(true)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('aborts a launch started near the deploy deadline and closes its channel once', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const launchChannel = {
|
||||
on: vi.fn(),
|
||||
stderr: { on: vi.fn() },
|
||||
stdin: {},
|
||||
stdout: { on: vi.fn() },
|
||||
close: vi.fn()
|
||||
}
|
||||
const conn = makeMockConnection()
|
||||
vi.mocked(conn.exec).mockResolvedValue(launchChannel as never)
|
||||
const mockExecCommand = vi.mocked(execCommand)
|
||||
mockExecCommand
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/user')
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<string>((resolve) =>
|
||||
setTimeout(() => resolve('ORCA-NATIVE-DEPS-OK'), 899_900)
|
||||
)
|
||||
)
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockImplementationOnce((_conn, _command, options) => {
|
||||
return new Promise<string>((_resolve, reject) => {
|
||||
options?.signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
const error = new Error('SSH operation was cancelled')
|
||||
error.name = 'AbortError'
|
||||
reject(error)
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.advanceTimersByTimeAsync(899_900)
|
||||
expect(conn.exec).toHaveBeenCalledTimes(1)
|
||||
expect(launchChannel.close).not.toHaveBeenCalled()
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100)
|
||||
|
||||
const result = await promise
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toBe('Relay deployment timed out after 900s')
|
||||
expect(launchChannel.close).toHaveBeenCalledTimes(1)
|
||||
expect(mockExecCommand).toHaveBeenCalledTimes(5)
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(mockExecCommand).toHaveBeenCalledTimes(5)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('uses distinct target-specific relay socket paths', async () => {
|
||||
const connA = makeMockConnection()
|
||||
const connB = makeMockConnection()
|
||||
|
|
@ -596,6 +768,20 @@ describe('deployAndLaunchRelay', () => {
|
|||
expect(launchScript).not.toContain('\\\\.\\pipe\\agent-hooks')
|
||||
const waitScript = decodedScripts.find((script) => script.includes('deadline=Date.now()')) ?? ''
|
||||
expect(waitScript).toContain('setTimeout(attempt,intervalMs)')
|
||||
const windowsLaunchCalls = mockExecCommand.mock.calls.filter(([, command]) => {
|
||||
const script = decodePowerShellCommand(command)
|
||||
return (
|
||||
script?.includes('.windows-active-pipe') ||
|
||||
script?.includes('Invoke-CimMethod') ||
|
||||
script?.includes('deadline=Date.now()')
|
||||
)
|
||||
})
|
||||
expect(windowsLaunchCalls.length).toBeGreaterThan(0)
|
||||
expect(
|
||||
windowsLaunchCalls.every(([, , options]) => options?.signal instanceof AbortSignal)
|
||||
).toBe(true)
|
||||
expect(vi.mocked(conn.exec).mock.calls[0]?.[1]?.signal).toBeInstanceOf(AbortSignal)
|
||||
expect(vi.mocked(waitForSentinel).mock.calls[0]?.[1]).toBeInstanceOf(AbortSignal)
|
||||
})
|
||||
|
||||
it('relaunches Windows remotes on a fallback pipe when reconnecting the occupied pipe fails', async () => {
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,156 @@
|
|||
import type { SshConnection } from './ssh-connection'
|
||||
import { createSshOperationAbortError, type SshExecOptions } from './ssh-connection-utils'
|
||||
import type { SystemSshCommandChannel } from './system-ssh-command'
|
||||
|
||||
const EXEC_TIMEOUT_MS = 30_000
|
||||
const COMMAND_CLOSE_GRACE_MS = 5_000
|
||||
const MAX_EXEC_OUTPUT_CHARS = 1024 * 1024
|
||||
|
||||
type ExecCommandOptions = SshExecOptions & {
|
||||
timeoutMs?: number
|
||||
}
|
||||
|
||||
type SshCommandTerminationError = Error & {
|
||||
sshChannelCloseConfirmed: boolean
|
||||
}
|
||||
|
||||
export function isUnconfirmedSshCommandTermination(
|
||||
error: unknown
|
||||
): error is SshCommandTerminationError {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
(error as Partial<SshCommandTerminationError>).sshChannelCloseConfirmed === false
|
||||
)
|
||||
}
|
||||
|
||||
export async function execCommand(
|
||||
conn: SshConnection,
|
||||
command: string,
|
||||
options?: ExecCommandOptions
|
||||
): Promise<string> {
|
||||
const { timeoutMs = EXEC_TIMEOUT_MS, ...execOptions } = options ?? {}
|
||||
const signal = options?.signal
|
||||
if (signal?.aborted) {
|
||||
throw createSshOperationAbortError()
|
||||
}
|
||||
// Why: reconnect/disconnect can flip the connection back to ssh2 before a
|
||||
// killed local OpenSSH child emits close; the channel's transport is immutable.
|
||||
const openedWithSystemSsh = conn.usesSystemSshTransport?.() === true
|
||||
const channel = await conn.exec(command, execOptions)
|
||||
return new Promise((resolve, reject) => {
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let settled = false
|
||||
let terminationError: SshCommandTerminationError | null = null
|
||||
let closeGraceTimer: ReturnType<typeof setTimeout> | null = null
|
||||
|
||||
const cleanup = (): void => {
|
||||
clearTimeout(timeout)
|
||||
if (closeGraceTimer) {
|
||||
clearTimeout(closeGraceTimer)
|
||||
}
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
channel.off('error', fail)
|
||||
channel.stderr.off('error', fail)
|
||||
channel.off('data', onStdoutData)
|
||||
channel.stderr.off('data', onStderrData)
|
||||
channel.off('close', onClose)
|
||||
}
|
||||
const settle = (fn: typeof resolve | typeof reject, val: string | Error): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
fn(val as never)
|
||||
}
|
||||
const guardUnconfirmedTeardown = (): void => {
|
||||
const swallowLateError = (): void => {}
|
||||
const cleanupGuards = (): void => {
|
||||
channel.off('error', swallowLateError)
|
||||
channel.stderr.off('error', swallowLateError)
|
||||
channel.off('close', cleanupGuards)
|
||||
}
|
||||
channel.on('error', swallowLateError)
|
||||
channel.stderr.on('error', swallowLateError)
|
||||
channel.once('close', cleanupGuards)
|
||||
// Why: an unconfirmed close can arrive after the caller's bounded wait;
|
||||
// keep draining discarded streams so ssh2 can finish CHANNEL_CLOSE.
|
||||
channel.resume()
|
||||
channel.stderr.resume()
|
||||
}
|
||||
// Why: sshd counts the session against MaxSessions until CHANNEL_CLOSE
|
||||
// completes. Settling on abort before the channel actually closes lets the
|
||||
// concurrent-bootstrap sequential fallback reissue an exec while the slot
|
||||
// is still held, so it gets refused again. Close and settle from onClose.
|
||||
const requestTermination = (error: Error): void => {
|
||||
if (terminationError) {
|
||||
return
|
||||
}
|
||||
terminationError = Object.assign(error, { sshChannelCloseConfirmed: false })
|
||||
clearTimeout(timeout)
|
||||
// Why: callers must not release an install lock while its remote npm
|
||||
// process can still mutate node_modules. Prefer confirmed channel close,
|
||||
// but bound a broken transport's teardown wait.
|
||||
closeGraceTimer = setTimeout(() => {
|
||||
guardUnconfirmedTeardown()
|
||||
settle(reject, error)
|
||||
}, COMMAND_CLOSE_GRACE_MS)
|
||||
channel.close()
|
||||
}
|
||||
const fail = (err: Error): void => requestTermination(err)
|
||||
const onAbort = (): void => requestTermination(createSshOperationAbortError())
|
||||
const onStdoutData = (data: Buffer): void => {
|
||||
stdout = appendExecOutputTail(stdout, data.toString('utf-8'))
|
||||
}
|
||||
const onStderrData = (data: Buffer): void => {
|
||||
stderr = appendExecOutputTail(stderr, data.toString('utf-8'))
|
||||
}
|
||||
const onClose = (code: number): void => {
|
||||
if (
|
||||
!terminationError &&
|
||||
openedWithSystemSsh &&
|
||||
(channel as SystemSshCommandChannel)._closeRequested
|
||||
) {
|
||||
terminationError = Object.assign(createSshOperationAbortError(), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
}
|
||||
if (terminationError) {
|
||||
// Why: a system-SSH channel closes when the local OpenSSH child exits;
|
||||
// that does not prove the remote command stopped, especially with a ControlMaster.
|
||||
if (!openedWithSystemSsh) {
|
||||
terminationError.sshChannelCloseConfirmed = true
|
||||
}
|
||||
settle(reject, terminationError)
|
||||
} else if (code !== 0) {
|
||||
// Why: on the system-ssh transport channel.stderr carries local OpenSSH
|
||||
// client noise; preferring it masks the real failure in stdout (2>&1).
|
||||
const output = [stderr.trim(), stdout.trim()].filter(Boolean).join('\n')
|
||||
settle(reject, new Error(`Command "${command}" failed (exit ${code}): ${output}`))
|
||||
} else {
|
||||
settle(resolve, stdout)
|
||||
}
|
||||
}
|
||||
const timeout = setTimeout(() => {
|
||||
requestTermination(new Error(`Command "${command}" timed out after ${timeoutMs / 1000}s`))
|
||||
}, timeoutMs)
|
||||
|
||||
// Why: remote reboot tears down exec channels with stream errors. Without
|
||||
// scoped listeners, Node treats those as uncaught exceptions.
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
channel.on('error', fail)
|
||||
channel.stderr.on('error', fail)
|
||||
channel.on('data', onStdoutData)
|
||||
channel.stderr.on('data', onStderrData)
|
||||
channel.on('close', onClose)
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function appendExecOutputTail(existing: string, chunk: string): string {
|
||||
const combined = existing + chunk
|
||||
return combined.length > MAX_EXEC_OUTPUT_CHARS ? combined.slice(-MAX_EXEC_OUTPUT_CHARS) : combined
|
||||
}
|
||||
|
|
@ -0,0 +1,236 @@
|
|||
import { randomUUID } from 'node:crypto'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import {
|
||||
probeInstallLockExistsCommand,
|
||||
tryCreateInstallLockCommand,
|
||||
tryStealInstallLockCommand
|
||||
} from './ssh-relay-install-lock-commands'
|
||||
import { removeRemoteTreeCommand } from './ssh-remote-commands'
|
||||
import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell'
|
||||
import {
|
||||
getRemoteHostPlatform,
|
||||
isWindowsRemoteHost,
|
||||
joinRemotePath,
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
|
||||
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')
|
||||
const RELAY_GC_CLAIM_SUFFIX = '.gc-claim'
|
||||
const RELAY_GC_OWNER_NAME = '.gc-owner'
|
||||
// Why: the claim guards only a bounded sibling rename or launch handoff, not
|
||||
// npm or deletion. Ten minutes bounds crashes while exceeding either sequence.
|
||||
const RELAY_GC_CLAIM_STALE_SECONDS = 10 * 60
|
||||
|
||||
function execHostCommand(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
command: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<string> {
|
||||
return execCommand(conn, command, {
|
||||
wrapCommand: !isWindowsRemoteHost(host),
|
||||
signal
|
||||
})
|
||||
}
|
||||
|
||||
export function relayGcClaimPath(remoteRelayDir: string): string {
|
||||
return `${remoteRelayDir}${RELAY_GC_CLAIM_SUFFIX}`
|
||||
}
|
||||
|
||||
export async function isRelayGcClaimed(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
signal?: AbortSignal
|
||||
): Promise<boolean> {
|
||||
const claimPath = relayGcClaimPath(remoteRelayDir)
|
||||
const output = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
probeInstallLockExistsCommand(host, claimPath),
|
||||
signal
|
||||
)
|
||||
const markers = new Set(
|
||||
output
|
||||
.split(/\r?\n/)
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line === 'LOCKED' || line === 'OPEN')
|
||||
)
|
||||
if (markers.size !== 1) {
|
||||
throw new Error(`Inconclusive relay GC claim probe at ${claimPath}`)
|
||||
}
|
||||
return markers.has('LOCKED')
|
||||
}
|
||||
|
||||
export async function tryAcquireRelayGcClaim(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
signal?: AbortSignal
|
||||
): Promise<string | null> {
|
||||
const claimPath = relayGcClaimPath(remoteRelayDir)
|
||||
try {
|
||||
const created = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryCreateInstallLockCommand(host, claimPath),
|
||||
signal
|
||||
)
|
||||
if (created.trim().endsWith('OK')) {
|
||||
return writeRelayGcClaimOwner(conn, remoteRelayDir, host, signal)
|
||||
}
|
||||
const stolen = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryStealInstallLockCommand(host, claimPath, RELAY_GC_CLAIM_STALE_SECONDS),
|
||||
signal
|
||||
)
|
||||
if (!stolen.trim().endsWith('OK')) {
|
||||
return null
|
||||
}
|
||||
return writeRelayGcClaimOwner(conn, remoteRelayDir, host, signal)
|
||||
} catch {
|
||||
signal?.throwIfAborted()
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function writeRelayGcClaimOwner(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform,
|
||||
signal?: AbortSignal
|
||||
): Promise<string | null> {
|
||||
const token = `${process.pid}-${Date.now()}-${randomUUID()}`
|
||||
const claimPath = relayGcClaimPath(remoteRelayDir)
|
||||
const ownerPath = joinRemotePath(host, claimPath, RELAY_GC_OWNER_NAME)
|
||||
const command = isWindowsRemoteHost(host)
|
||||
? powerShellCommand(
|
||||
`Set-Content -LiteralPath ${powerShellLiteral(ownerPath)} -Value ${powerShellLiteral(token)} -NoNewline -ErrorAction Stop`
|
||||
)
|
||||
: `printf %s ${shellEscape(token)} > ${shellEscape(ownerPath)}`
|
||||
try {
|
||||
await execHostCommand(conn, host, command, signal)
|
||||
return token
|
||||
} catch {
|
||||
// Why: the write may have succeeded remotely before SSH lost its reply.
|
||||
// A conditional release removes only the claim generation with our token.
|
||||
await releaseRelayGcClaim(conn, remoteRelayDir, token, host)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export async function isRelayGcClaimOwned(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
token: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<boolean> {
|
||||
const ownerPath = joinRemotePath(host, relayGcClaimPath(remoteRelayDir), RELAY_GC_OWNER_NAME)
|
||||
const command = isWindowsRemoteHost(host)
|
||||
? powerShellCommand(
|
||||
`if ((Get-Content -LiteralPath ${powerShellLiteral(ownerPath)} -Raw -ErrorAction SilentlyContinue) -ceq ${powerShellLiteral(token)}) { 'OWNED' } else { 'LOST' }`
|
||||
)
|
||||
: `test "$(cat ${shellEscape(ownerPath)} 2>/dev/null)" = ${shellEscape(token)} && echo OWNED || echo LOST`
|
||||
const output = await execHostCommand(conn, host, command).catch(() => 'LOST')
|
||||
return output.trim() === 'OWNED'
|
||||
}
|
||||
|
||||
export type RelayGcClaimReleaseResult = 'released' | 'lost' | 'unknown'
|
||||
|
||||
export async function releaseRelayGcClaim(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
token: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<RelayGcClaimReleaseResult> {
|
||||
const claimPath = relayGcClaimPath(remoteRelayDir)
|
||||
const ownerPath = joinRemotePath(host, claimPath, RELAY_GC_OWNER_NAME)
|
||||
const command = isWindowsRemoteHost(host)
|
||||
? powerShellCommand(
|
||||
`$claim = ${powerShellLiteral(claimPath)}; ` +
|
||||
`if (-not (Test-Path -LiteralPath $claim)) { 'RELEASED' } ` +
|
||||
`elseif ((Get-Content -LiteralPath ${powerShellLiteral(ownerPath)} -Raw -ErrorAction SilentlyContinue) -cne ${powerShellLiteral(token)}) { 'LOST' } ` +
|
||||
'else { try { Remove-Item -LiteralPath $claim -Recurse -Force -ErrorAction Stop } catch {}; ' +
|
||||
"if (Test-Path -LiteralPath $claim) { 'UNKNOWN' } else { 'RELEASED' } }"
|
||||
)
|
||||
: [
|
||||
`if ! test -e ${shellEscape(claimPath)}; then echo RELEASED;`,
|
||||
`elif test "$(cat ${shellEscape(ownerPath)} 2>/dev/null)" != ${shellEscape(token)}; then echo LOST;`,
|
||||
`else ${removeRemoteTreeCommand(host, claimPath)} 2>/dev/null;`,
|
||||
`if test -e ${shellEscape(claimPath)}; then echo UNKNOWN; else echo RELEASED; fi; fi`
|
||||
].join(' ')
|
||||
const output = await execHostCommand(conn, host, command).catch(() => 'UNKNOWN')
|
||||
switch (output.trim()) {
|
||||
case 'RELEASED':
|
||||
return 'released'
|
||||
case 'LOST':
|
||||
return 'lost'
|
||||
default:
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
export async function releaseRelayGcClaimWithRetry(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
token: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<RelayGcClaimReleaseResult> {
|
||||
let result: RelayGcClaimReleaseResult = 'unknown'
|
||||
for (let attempt = 0; attempt < 3 && result === 'unknown'; attempt++) {
|
||||
result = await releaseRelayGcClaim(conn, remoteRelayDir, token, host)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export async function waitForRelayGcClaimRelease(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
signal?: AbortSignal
|
||||
): Promise<void> {
|
||||
while (true) {
|
||||
const claimed = await isRelayGcClaimed(conn, remoteRelayDir, host, signal).catch(() => true)
|
||||
signal?.throwIfAborted()
|
||||
if (!claimed) {
|
||||
return
|
||||
}
|
||||
const claimPath = relayGcClaimPath(remoteRelayDir)
|
||||
const recovered = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryStealInstallLockCommand(host, claimPath, RELAY_GC_CLAIM_STALE_SECONDS),
|
||||
signal
|
||||
).catch(() => 'BUSY')
|
||||
signal?.throwIfAborted()
|
||||
if (recovered.trim().endsWith('OK')) {
|
||||
const token = await writeRelayGcClaimOwner(conn, remoteRelayDir, host, signal)
|
||||
if (token) {
|
||||
const release = await releaseRelayGcClaim(conn, remoteRelayDir, token, host)
|
||||
if (release === 'released') {
|
||||
return
|
||||
}
|
||||
}
|
||||
// A lost or uncertain release can mean a successor owns the stable path.
|
||||
// Probe again instead of treating recovery as complete.
|
||||
continue
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const onAbort = (): void => {
|
||||
clearTimeout(timer)
|
||||
reject(signal?.reason)
|
||||
}
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', onAbort)
|
||||
resolve()
|
||||
}, 1_000)
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
if (signal?.aborted) {
|
||||
onAbort()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,275 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('electron', () => ({ app: { getAppPath: () => '/mock/app' } }))
|
||||
vi.mock('fs', () => ({
|
||||
existsSync: vi.fn().mockReturnValue(true),
|
||||
readFileSync: vi.fn().mockReturnValue('0.1.0+gc-retry')
|
||||
}))
|
||||
vi.mock('./relay-protocol', () => ({
|
||||
RELAY_VERSION: '0.1.0',
|
||||
RELAY_REMOTE_DIR: '.orca-remote',
|
||||
parseUnameToRelayPlatform: vi.fn().mockReturnValue('linux-x64'),
|
||||
RELAY_SENTINEL: 'ORCA-RELAY v0.1.0 READY\n',
|
||||
RELAY_SENTINEL_TIMEOUT_MS: 10_000
|
||||
}))
|
||||
vi.mock('./ssh-relay-deploy-helpers', () => ({
|
||||
uploadDirectory: vi.fn(),
|
||||
waitForSentinel: vi.fn().mockResolvedValue({
|
||||
write: vi.fn(),
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}),
|
||||
isUnconfirmedSshCommandTermination: () => false,
|
||||
execCommand: vi.fn()
|
||||
}))
|
||||
vi.mock('./ssh-remote-node-resolution', () => ({
|
||||
resolveRemoteNodePath: vi.fn().mockResolvedValue('/usr/bin/node')
|
||||
}))
|
||||
vi.mock('./ssh-relay-versioned-install', () => ({
|
||||
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+gc-retry'),
|
||||
computeRemoteRelayDir: (home: string, version: string) => `${home}/.orca-remote/relay-${version}`,
|
||||
isRelayAlreadyInstalled: vi.fn().mockResolvedValue(true),
|
||||
finalizeInstall: vi.fn(),
|
||||
abandonInstall: vi.fn(),
|
||||
gcOldRelayVersions: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
vi.mock('./ssh-relay-install-lock', () => ({ acquireInstallLock: vi.fn() }))
|
||||
vi.mock('./ssh-relay-repair-lock', () => ({
|
||||
tryAcquireRelayRepairLock: vi.fn().mockResolvedValue('acquired')
|
||||
}))
|
||||
vi.mock('./ssh-relay-gc-claim', () => ({
|
||||
releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'),
|
||||
tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('launch-token'),
|
||||
waitForRelayGcClaimRelease: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
vi.mock('./ssh-connection-utils', () => ({
|
||||
shellEscape: (value: string) => `'${value}'`,
|
||||
createSshOperationAbortError: () => Object.assign(new Error('cancelled'), { name: 'AbortError' })
|
||||
}))
|
||||
|
||||
import { deployAndLaunchRelay } from './ssh-relay-deploy'
|
||||
import { execCommand, waitForSentinel } from './ssh-relay-deploy-helpers'
|
||||
import {
|
||||
releaseRelayGcClaimWithRetry,
|
||||
tryAcquireRelayGcClaim,
|
||||
waitForRelayGcClaimRelease
|
||||
} from './ssh-relay-gc-claim'
|
||||
import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock'
|
||||
import { abandonInstall, isRelayAlreadyInstalled } from './ssh-relay-versioned-install'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
|
||||
function makeConnection(): SshConnection {
|
||||
const channel = {
|
||||
on: vi.fn(),
|
||||
stderr: { on: vi.fn() },
|
||||
close: vi.fn()
|
||||
}
|
||||
return {
|
||||
canRunConcurrentExecCommands: vi.fn().mockReturnValue(true),
|
||||
exec: vi.fn().mockResolvedValue(channel)
|
||||
} as unknown as SshConnection
|
||||
}
|
||||
|
||||
describe('relay GC deploy retry', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('recomputes install state when GC wins before a healthy relay launch', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce('gc')
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(waitForRelayGcClaimRelease).toHaveBeenCalledTimes(1)
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(3)
|
||||
expect(conn.exec).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('recomputes state when GC finishes between the install probe and lock acquisition', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(isRelayAlreadyInstalled)
|
||||
.mockResolvedValueOnce(true)
|
||||
.mockResolvedValueOnce(false)
|
||||
.mockResolvedValue(true)
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(abandonInstall).toHaveBeenCalledTimes(2)
|
||||
expect(waitForRelayGcClaimRelease).toHaveBeenCalledTimes(1)
|
||||
expect(isRelayAlreadyInstalled).toHaveBeenCalledTimes(4)
|
||||
expect(tryAcquireRelayRepairLock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('owns a GC claim through launch when the install-lock state is indeterminate', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce('error')
|
||||
let resolveReady: ((value: Awaited<ReturnType<typeof waitForSentinel>>) => void) | undefined
|
||||
vi.mocked(waitForSentinel).mockImplementationOnce(
|
||||
async () =>
|
||||
new Promise<Awaited<ReturnType<typeof waitForSentinel>>>((resolve) => {
|
||||
resolveReady = resolve
|
||||
})
|
||||
)
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const deploy = deployAndLaunchRelay(conn)
|
||||
await vi.waitFor(() => expect(resolveReady).toBeDefined())
|
||||
expect(releaseRelayGcClaimWithRetry).not.toHaveBeenCalled()
|
||||
resolveReady?.({
|
||||
write: vi.fn(),
|
||||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
})
|
||||
await deploy
|
||||
|
||||
expect(waitForRelayGcClaimRelease).not.toHaveBeenCalled()
|
||||
expect(tryAcquireRelayRepairLock).toHaveBeenCalledTimes(1)
|
||||
expect(tryAcquireRelayGcClaim).toHaveBeenCalledTimes(1)
|
||||
expect(releaseRelayGcClaimWithRetry).toHaveBeenCalledTimes(1)
|
||||
expect(conn.exec).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('retains an owned install lock when launch never becomes live', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('launch failed'))
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
await expect(deployAndLaunchRelay(conn)).rejects.toThrow('launch failed')
|
||||
|
||||
expect(abandonInstall).not.toHaveBeenCalled()
|
||||
expect(releaseRelayGcClaimWithRetry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('retains an owned launch claim when launch never becomes live', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce('busy')
|
||||
vi.mocked(waitForSentinel).mockRejectedValueOnce(new Error('launch failed'))
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
await expect(deployAndLaunchRelay(conn)).rejects.toThrow('launch failed')
|
||||
|
||||
expect(abandonInstall).not.toHaveBeenCalled()
|
||||
expect(releaseRelayGcClaimWithRetry).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps retrying repeated launch-claim contention within the deploy bound', async () => {
|
||||
const conn = makeConnection()
|
||||
vi.mocked(tryAcquireRelayRepairLock).mockResolvedValue('busy')
|
||||
vi.mocked(tryAcquireRelayGcClaim)
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValueOnce(null)
|
||||
.mockResolvedValue('launch-token')
|
||||
vi.mocked(execCommand).mockImplementation(async (_conn, command) => {
|
||||
if (command.includes('__ORCA_REMOTE_PLATFORM__')) {
|
||||
return '__ORCA_REMOTE_PLATFORM__ Linux x86_64'
|
||||
}
|
||||
if (command === 'echo $HOME') {
|
||||
return '/home/user'
|
||||
}
|
||||
if (command.includes('node-pty')) {
|
||||
return 'ORCA-NATIVE-DEPS-OK'
|
||||
}
|
||||
if (command.includes('var s=require("net").connect')) {
|
||||
return 'READY'
|
||||
}
|
||||
if (command.includes('test -S')) {
|
||||
return 'DEAD'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(waitForRelayGcClaimRelease).toHaveBeenCalledTimes(2)
|
||||
expect(tryAcquireRelayGcClaim).toHaveBeenCalledTimes(3)
|
||||
expect(releaseRelayGcClaimWithRetry).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import type { SshConnection } from './ssh-connection'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { removeRemoteTreeCommand } from './ssh-remote-commands'
|
||||
import {
|
||||
getRemoteHostPlatform,
|
||||
isWindowsRemoteHost,
|
||||
joinRemotePath,
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
|
||||
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')
|
||||
const RELAY_GC_TOMBSTONE_REGEX =
|
||||
/^relay-(?:v?\d+\.\d+\.\d+(?:\+[0-9a-f]+)?)\.gc-tombstone\.\d+\.\d+$/
|
||||
|
||||
export async function cleanupRelayGcTombstones(
|
||||
conn: SshConnection,
|
||||
baseDir: string,
|
||||
entries: string[],
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<void> {
|
||||
// Why: a confirmed rename isolates these paths from any recreated install.
|
||||
// Strictly named leftovers are safe to retry after an interrupted GC pass.
|
||||
for (const name of entries.filter((entry) => RELAY_GC_TOMBSTONE_REGEX.test(entry))) {
|
||||
const tombstone = joinRemotePath(host, baseDir, name)
|
||||
await execCommand(conn, removeRemoteTreeCommand(host, tombstone), {
|
||||
wrapCommand: !isWindowsRemoteHost(host)
|
||||
}).catch((err) => {
|
||||
console.warn(
|
||||
`[ssh-relay] GC failed to remove tombstone ${tombstone}: ${err instanceof Error ? err.message : String(err)}`
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
import { shellEscape } from './ssh-connection-utils'
|
||||
import { powerShellCommand, powerShellLiteral } from './ssh-remote-powershell'
|
||||
import { isWindowsRemoteHost, type RemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
export function acquireInstallLockParentCommand(
|
||||
host: RemoteHostPlatform,
|
||||
remoteRelayDir: string
|
||||
): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `mkdir -p ${shellEscape(remoteRelayDir)}`
|
||||
}
|
||||
return powerShellCommand(
|
||||
`$null = New-Item -ItemType Directory -Force -Path ${powerShellLiteral(remoteRelayDir)}`
|
||||
)
|
||||
}
|
||||
|
||||
export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `mkdir ${shellEscape(lockDir)} 2>&1 && echo OK || echo BUSY`
|
||||
}
|
||||
// Why: old Orca clients recognize only a directory at `.install-lock`, while
|
||||
// concurrent New-Item calls can both report success in PowerShell 5.1. Keep
|
||||
// that directory marker and arbitrate ownership with an atomic child file.
|
||||
return powerShellCommand(
|
||||
[
|
||||
`$lock = ${powerShellLiteral(lockDir)}`,
|
||||
'$stream = $null',
|
||||
'try {',
|
||||
"if (Test-Path -LiteralPath $lock) { 'BUSY' } else {",
|
||||
'$null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop',
|
||||
"$owner = Join-Path $lock '.owner'",
|
||||
'$stream = [System.IO.File]::Open($owner, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)',
|
||||
"'OK'",
|
||||
'}',
|
||||
`} catch { 'BUSY' } finally { if ($null -ne $stream) { $stream.Dispose() } }`
|
||||
].join('; ')
|
||||
)
|
||||
}
|
||||
|
||||
export function probeInstallLockExistsCommand(host: RemoteHostPlatform, lockPath: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `test -e ${shellEscape(lockPath)} && echo LOCKED || echo OPEN`
|
||||
}
|
||||
// Why: one prerelease briefly wrote file locks; accept both shapes so those
|
||||
// hosts remain recoverable after upgrading to directory-plus-owner locks.
|
||||
return powerShellCommand(
|
||||
`if (Test-Path -LiteralPath ${powerShellLiteral(lockPath)}) { 'LOCKED' } else { 'OPEN' }`
|
||||
)
|
||||
}
|
||||
|
||||
export function lockAgeSecondsCommand(host: RemoteHostPlatform, lockDir: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `${posixLockAgeSecondsAssignment(lockDir)} && echo "$age" || echo`
|
||||
}
|
||||
return powerShellCommand(
|
||||
[
|
||||
`$item = Get-Item -LiteralPath ${powerShellLiteral(lockDir)} -ErrorAction Stop`,
|
||||
'$mtime = ([DateTimeOffset]$item.LastWriteTimeUtc).ToUnixTimeSeconds()',
|
||||
'$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()',
|
||||
'Write-Output ($now - $mtime)'
|
||||
].join('; ')
|
||||
)
|
||||
}
|
||||
|
||||
export function tryStealInstallLockCommand(
|
||||
host: RemoteHostPlatform,
|
||||
lockDir: string,
|
||||
staleAfterSeconds: number
|
||||
): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return posixStealInstallLockCommand(lockDir, staleAfterSeconds)
|
||||
}
|
||||
return windowsStealInstallLockCommand(lockDir, staleAfterSeconds)
|
||||
}
|
||||
|
||||
function posixStealInstallLockCommand(lockDir: string, staleAfterSeconds: number): string {
|
||||
const escapedLockDir = shellEscape(lockDir)
|
||||
const escapedStealLockPrefix = shellEscape(`${lockDir}.steal`)
|
||||
return [
|
||||
`${posixLockIdentityAssignment(lockDir, 'lock_key')} && mtime=\${lock_key%%:*} && now=$(date +%s) && age=$((now - mtime)) || age=0;`,
|
||||
`if [ "\${age:-0}" -le ${staleAfterSeconds} ] 2>/dev/null; then echo BUSY; else`,
|
||||
`steal_root=${escapedStealLockPrefix};`,
|
||||
'steal_generation=0;',
|
||||
'steal="$steal_root.$steal_generation";',
|
||||
'owns_steal=0;',
|
||||
'lock_tombstone=;',
|
||||
'while [ "$owns_steal" != 1 ]; do',
|
||||
'if mkdir "$steal" 2>/dev/null; then owns_steal=1; break; fi;',
|
||||
`steal_mtime=$(stat -c %Y "$steal" 2>/dev/null || stat -f %m "$steal" 2>/dev/null) && steal_now=$(date +%s) && steal_age=$((steal_now - steal_mtime)) || break;`,
|
||||
'if [ "${steal_age:-0}" -le 120 ] 2>/dev/null; then break; fi;',
|
||||
'steal_generation=$((steal_generation + 1));',
|
||||
'steal="$steal_root.$steal_generation";',
|
||||
'done;',
|
||||
'if [ "$owns_steal" = 1 ]; then',
|
||||
`trap 'rm -rf "$steal_root".* 2>/dev/null || true; rm -rf "$lock_tombstone" 2>/dev/null || true' EXIT;`,
|
||||
`${posixLockIdentityAssignment(lockDir, 'current_key')} && current_mtime=\${current_key%%:*} && current_now=$(date +%s) && current_age=$((current_now - current_mtime)) || current_age=0;`,
|
||||
`if [ "$current_key" = "$lock_key" ] && [ "\${current_age:-0}" -gt ${staleAfterSeconds} ] 2>/dev/null; then`,
|
||||
`lock_tombstone=${escapedLockDir}.tombstone.$$.$(date +%s);`,
|
||||
`if [ ! -e "$lock_tombstone" ] && mv ${escapedLockDir} "$lock_tombstone" 2>/dev/null; then mkdir ${escapedLockDir} 2>&1 && echo OK || echo BUSY; else echo BUSY; fi;`,
|
||||
'else echo BUSY; fi;',
|
||||
'else echo BUSY; fi; fi'
|
||||
].join(' ')
|
||||
}
|
||||
|
||||
function windowsStealInstallLockCommand(lockDir: string, staleAfterSeconds: number): string {
|
||||
return powerShellCommand(
|
||||
[
|
||||
`$lock = ${powerShellLiteral(lockDir)}`,
|
||||
'try {',
|
||||
'$item = Get-Item -LiteralPath $lock -ErrorAction Stop',
|
||||
'$mtime = ([DateTimeOffset]$item.LastWriteTimeUtc).ToUnixTimeSeconds()',
|
||||
'$lockIdentity = "${mtime}:$($item.CreationTimeUtc.Ticks)"',
|
||||
'$now = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()',
|
||||
`if (($now - $mtime) -le ${staleAfterSeconds}) { 'BUSY' } else {`,
|
||||
'$stealRoot = "$lock.steal"',
|
||||
'$stealGeneration = 0',
|
||||
'$steal = "$stealRoot.$stealGeneration"',
|
||||
'$ownsSteal = $false',
|
||||
'$lockTombstone = $null',
|
||||
'try {',
|
||||
'while (-not $ownsSteal) {',
|
||||
'try {',
|
||||
'$stealStream = [System.IO.File]::Open($steal, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None)',
|
||||
'$stealStream.Dispose()',
|
||||
'$ownsSteal = $true',
|
||||
'break',
|
||||
'} catch {',
|
||||
'try {',
|
||||
'$stealItem = Get-Item -LiteralPath $steal -ErrorAction Stop',
|
||||
'$stealMtime = ([DateTimeOffset]$stealItem.LastWriteTimeUtc).ToUnixTimeSeconds()',
|
||||
'$stealAge = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() - $stealMtime',
|
||||
'if ($stealAge -le 120) { break }',
|
||||
'$stealGeneration++',
|
||||
'$steal = "$stealRoot.$stealGeneration"',
|
||||
'} catch { break }',
|
||||
'}',
|
||||
'}',
|
||||
"if (-not $ownsSteal) { 'BUSY' } else {",
|
||||
'$current = Get-Item -LiteralPath $lock -ErrorAction Stop',
|
||||
'$currentMtime = ([DateTimeOffset]$current.LastWriteTimeUtc).ToUnixTimeSeconds()',
|
||||
'$currentIdentity = "${currentMtime}:$($current.CreationTimeUtc.Ticks)"',
|
||||
'$currentNow = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()',
|
||||
`if (($currentIdentity -eq $lockIdentity) -and (($currentNow - $currentMtime) -gt ${staleAfterSeconds})) {`,
|
||||
'$lockTombstone = "$lock.tombstone.$PID.$([DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds())"',
|
||||
'Move-Item -LiteralPath $lock -Destination $lockTombstone -ErrorAction Stop',
|
||||
'$successorStream = $null',
|
||||
"try { $null = New-Item -ItemType Directory -Path $lock -ErrorAction Stop; $successorOwner = Join-Path $lock '.owner'; $successorStream = [System.IO.File]::Open($successorOwner, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write, [System.IO.FileShare]::None); 'OK' } catch { 'BUSY' } finally { if ($null -ne $successorStream) { $successorStream.Dispose() } }",
|
||||
"} else { 'BUSY' }",
|
||||
'}',
|
||||
"} catch { 'BUSY' } finally {",
|
||||
'if ($ownsSteal) {',
|
||||
'$stealParent = Split-Path -Parent $stealRoot',
|
||||
'$stealLeaf = Split-Path -Leaf $stealRoot',
|
||||
'Get-ChildItem -LiteralPath $stealParent -Force -ErrorAction SilentlyContinue | Where-Object { $_.Name.StartsWith($stealLeaf + ".", [StringComparison]::Ordinal) } | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue',
|
||||
'}',
|
||||
'if ($null -ne $lockTombstone) { Remove-Item -LiteralPath $lockTombstone -Recurse -Force -ErrorAction SilentlyContinue }',
|
||||
'}',
|
||||
'}',
|
||||
"} catch { 'BUSY' }"
|
||||
].join('; ')
|
||||
)
|
||||
}
|
||||
|
||||
function posixLockAgeSecondsAssignment(lockDir: string): string {
|
||||
return `${posixLockMtimeSecondsAssignment(lockDir, 'mtime')} && now=$(date +%s) && age=$((now - mtime))`
|
||||
}
|
||||
|
||||
function posixLockIdentityAssignment(lockDir: string, variableName: string): string {
|
||||
const escapedLockDir = shellEscape(lockDir)
|
||||
return `${variableName}=$(stat -c %Y:%i ${escapedLockDir} 2>/dev/null || stat -f %m:%i ${escapedLockDir} 2>/dev/null)`
|
||||
}
|
||||
|
||||
function posixLockMtimeSecondsAssignment(lockDir: string, variableName: string): string {
|
||||
const escapedLockDir = shellEscape(lockDir)
|
||||
return `${variableName}=$(stat -c %Y ${escapedLockDir} 2>/dev/null || stat -f %m ${escapedLockDir} 2>/dev/null)`
|
||||
}
|
||||
|
|
@ -0,0 +1,162 @@
|
|||
import type { SshConnection } from './ssh-connection'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
|
||||
import { isRelayGcClaimed, waitForRelayGcClaimRelease } from './ssh-relay-gc-claim'
|
||||
import {
|
||||
acquireInstallLockParentCommand,
|
||||
lockAgeSecondsCommand,
|
||||
tryCreateInstallLockCommand,
|
||||
tryStealInstallLockCommand
|
||||
} from './ssh-relay-install-lock-commands'
|
||||
import {
|
||||
getRemoteHostPlatform,
|
||||
joinRemotePath,
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
import { removeRemoteTreeCommand } from './ssh-remote-commands'
|
||||
|
||||
export const RELAY_INSTALL_LOCK_NAME = '.install-lock'
|
||||
|
||||
const INSTALL_LOCK_POLL_MS = 1_000
|
||||
// Why: a fresh lock can cross the stale threshold during our bounded wait.
|
||||
// Recheck infrequently so it becomes recoverable without adding an exec per poll.
|
||||
const INSTALL_LOCK_STALE_RECHECK_MS = 60_000
|
||||
// Why: the lock holder can legitimately use the full deploy bound for upload,
|
||||
// native install/rebuild, probes, and finalization. A concurrent first install
|
||||
// must not fail earlier; completed-relay repair uses a separate one-shot path.
|
||||
const INSTALL_LOCK_TIMEOUT_MS = RELAY_DEPLOY_TIMEOUT_MS
|
||||
// Why: native-deps repair can keep running after the deploy backstop wins its
|
||||
// Promise.race, so stale takeover must leave room for that bounded work.
|
||||
export const INSTALL_LOCK_STALE_MS = 20 * 60_000
|
||||
export const INSTALL_LOCK_STALE_SECONDS = INSTALL_LOCK_STALE_MS / 1000
|
||||
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')
|
||||
|
||||
function execHostCommand(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
command: string,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<string> {
|
||||
return execCommand(conn, command, {
|
||||
wrapCommand: host.commandDialect !== 'powershell',
|
||||
signal: options?.signal
|
||||
})
|
||||
}
|
||||
|
||||
export async function isRelayInstallLockStale(
|
||||
conn: SshConnection,
|
||||
lockDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
// Why: remote time avoids clock skew between Orca clients making a live
|
||||
// repair lock look old enough for GC or another installer to recover.
|
||||
const out = await execHostCommand(conn, host, lockAgeSecondsCommand(host, lockDir))
|
||||
const ageSec = Number.parseInt(out.trim(), 10)
|
||||
return Number.isFinite(ageSec) && ageSec >= 0 && ageSec * 1000 > INSTALL_LOCK_STALE_MS
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire the per-version install lock via a host-native exclusive create.
|
||||
* Why: POSIX mkdir and a Windows directory plus atomic owner file each give
|
||||
* one winner while keeping the marker visible to older Windows Orca clients.
|
||||
*/
|
||||
export async function acquireInstallLock(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<void> {
|
||||
const lockDir = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
|
||||
|
||||
const start = Date.now()
|
||||
let lastStaleCheckAt = Number.NEGATIVE_INFINITY
|
||||
while (true) {
|
||||
// Why: a crashed GC can leave the stable sibling claim behind. The shared
|
||||
// waiter recovers stale claims instead of polling that orphan forever.
|
||||
await waitForRelayGcClaimRelease(conn, remoteRelayDir, host, options?.signal)
|
||||
options?.signal?.throwIfAborted()
|
||||
await execHostCommand(conn, host, acquireInstallLockParentCommand(host, remoteRelayDir), {
|
||||
signal: options?.signal
|
||||
})
|
||||
try {
|
||||
const result = await execHostCommand(conn, host, tryCreateInstallLockCommand(host, lockDir), {
|
||||
signal: options?.signal
|
||||
})
|
||||
if (result.trim().endsWith('OK')) {
|
||||
// Why: GC may claim the sibling path between our first probe and lock
|
||||
// creation. Recheck while holding the in-tree lock; one side backs off.
|
||||
const claimedAfterAcquire = await isRelayGcClaimed(
|
||||
conn,
|
||||
remoteRelayDir,
|
||||
host,
|
||||
options?.signal
|
||||
).catch(() => true)
|
||||
if (!claimedAfterAcquire && !options?.signal?.aborted) {
|
||||
return
|
||||
}
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {})
|
||||
options?.signal?.throwIfAborted()
|
||||
}
|
||||
} catch {
|
||||
options?.signal?.throwIfAborted()
|
||||
// A failed mkdir is lock contention; keep the connection-specific error
|
||||
// out of the user path until the bounded wait expires.
|
||||
}
|
||||
if (Date.now() - lastStaleCheckAt >= INSTALL_LOCK_STALE_RECHECK_MS) {
|
||||
lastStaleCheckAt = Date.now()
|
||||
// Why: recover an already-stale lock immediately, then keep checking in
|
||||
// case a fresh holder crosses the stale threshold while we are waiting.
|
||||
const steal = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryStealInstallLockCommand(host, lockDir, INSTALL_LOCK_STALE_SECONDS),
|
||||
{ signal: options?.signal }
|
||||
).catch(() => 'BUSY')
|
||||
options?.signal?.throwIfAborted()
|
||||
if (steal.trim().endsWith('OK')) {
|
||||
console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`)
|
||||
const claimedAfterSteal = await isRelayGcClaimed(
|
||||
conn,
|
||||
remoteRelayDir,
|
||||
host,
|
||||
options?.signal
|
||||
).catch(() => true)
|
||||
if (!claimedAfterSteal && !options?.signal?.aborted) {
|
||||
return
|
||||
}
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {})
|
||||
options?.signal?.throwIfAborted()
|
||||
}
|
||||
}
|
||||
if (Date.now() - start >= INSTALL_LOCK_TIMEOUT_MS) {
|
||||
throw new Error(
|
||||
`Could not acquire relay install lock at ${lockDir} after ${
|
||||
INSTALL_LOCK_TIMEOUT_MS / 1000
|
||||
}s; another install is still in progress.`
|
||||
)
|
||||
}
|
||||
await waitForInstallLockPoll(options?.signal)
|
||||
}
|
||||
}
|
||||
|
||||
function waitForInstallLockPoll(signal?: AbortSignal): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const handleAbort = (): void => {
|
||||
clearTimeout(timeout)
|
||||
signal?.removeEventListener('abort', handleAbort)
|
||||
reject(signal?.reason)
|
||||
}
|
||||
const timeout = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', handleAbort)
|
||||
resolve()
|
||||
}, INSTALL_LOCK_POLL_MS)
|
||||
signal?.addEventListener('abort', handleAbort, { once: true })
|
||||
if (signal?.aborted) {
|
||||
handleAbort()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
// Why: regression coverage for the install-probe contract. The original
|
||||
// "node-pty is not available" bug shipped because every layer that should
|
||||
// have caught it (chained shell, swallowing catch, dir-only probe) was
|
||||
// have caught it (chained shell, swallowing catch, resolve-only probe) was
|
||||
// silent. Tests below pin the parts that, individually, would have caught
|
||||
// it.
|
||||
|
||||
|
|
@ -30,6 +30,9 @@ vi.mock('./ssh-relay-deploy-helpers', () => ({
|
|||
onData: vi.fn(),
|
||||
onClose: vi.fn()
|
||||
}),
|
||||
isUnconfirmedSshCommandTermination: (error: unknown) =>
|
||||
error instanceof Error &&
|
||||
(error as Error & { sshChannelCloseConfirmed?: boolean }).sshChannelCloseConfirmed === false,
|
||||
execCommand: vi.fn()
|
||||
}))
|
||||
|
||||
|
|
@ -41,26 +44,41 @@ vi.mock('./ssh-relay-versioned-install', () => ({
|
|||
readLocalFullVersion: vi.fn().mockReturnValue('0.1.0+testhash'),
|
||||
computeRemoteRelayDir: (home: string, v: string) => `${home}/.orca-remote/relay-${v}`,
|
||||
isRelayAlreadyInstalled: vi.fn().mockResolvedValue(false),
|
||||
acquireInstallLock: vi.fn().mockResolvedValue(undefined),
|
||||
finalizeInstall: vi.fn().mockResolvedValue(undefined),
|
||||
abandonInstall: vi.fn().mockResolvedValue(undefined),
|
||||
gcOldRelayVersions: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-install-lock', () => ({
|
||||
acquireInstallLock: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-repair-lock', () => ({
|
||||
tryAcquireRelayRepairLock: vi.fn().mockResolvedValue('acquired')
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-relay-gc-claim', () => ({
|
||||
releaseRelayGcClaimWithRetry: vi.fn().mockResolvedValue('released'),
|
||||
tryAcquireRelayGcClaim: vi.fn().mockResolvedValue('launch-token'),
|
||||
waitForRelayGcClaimRelease: vi.fn().mockResolvedValue(undefined)
|
||||
}))
|
||||
|
||||
vi.mock('./ssh-connection-utils', () => ({
|
||||
shellEscape: (s: string) => `'${s}'`
|
||||
}))
|
||||
|
||||
import { deployAndLaunchRelay } from './ssh-relay-deploy'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { execCommand, uploadDirectory } from './ssh-relay-deploy-helpers'
|
||||
import { RELAY_DEPLOY_TIMEOUT_MS } from './ssh-relay-deploy-timing'
|
||||
import { parseUnameToRelayPlatform } from './relay-protocol'
|
||||
import { resolveRemoteNodePath } from './ssh-remote-node-resolution'
|
||||
import {
|
||||
acquireInstallLock,
|
||||
abandonInstall,
|
||||
finalizeInstall,
|
||||
isRelayAlreadyInstalled
|
||||
} from './ssh-relay-versioned-install'
|
||||
import { acquireInstallLock } from './ssh-relay-install-lock'
|
||||
import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
|
||||
type SftpWriteCapture = {
|
||||
|
|
@ -129,6 +147,7 @@ function decodePowerShellCommand(command: string): string | null {
|
|||
// 7: probe (cd && node -e require)
|
||||
// [8: cat stderr — only when probe stdout is MISSING (graceful path)]
|
||||
// 8 or 9: rm probe-stderr (best-effort cleanup; runs whenever probe resolved)
|
||||
// [next: npm rebuild → chmod → second probe when the first probe is MISSING]
|
||||
// next: socket DEAD next: socket READY
|
||||
//
|
||||
// When the probe rejects (SSH channel close or cd-failure when the install
|
||||
|
|
@ -143,6 +162,9 @@ function makeExecResponses(opts: {
|
|||
// Override probe stdout for shell-noise pressure tests. If set, replaces
|
||||
// the load-test stdout entirely (useful for testing pollution prefixes).
|
||||
probeStdoutOverride?: string
|
||||
// Result after the automatic rebuild. Defaults to missing so legacy tests
|
||||
// continue to exercise the final degraded-mode warning.
|
||||
repairProbe?: 'ok' | 'missing'
|
||||
// Raw stdout for the build-toolchain probe that runs in installNativeDeps'
|
||||
// catch when `npm install` rejects on Linux. Defaults to a fully-present
|
||||
// toolchain so the original npm error propagates unchanged.
|
||||
|
|
@ -187,6 +209,16 @@ function makeExecResponses(opts: {
|
|||
slots.push('') // cat stderr (graceful failure path captures detail)
|
||||
}
|
||||
slots.push('') // rm -f stderr (best-effort cleanup)
|
||||
if (!probeOk) {
|
||||
slots.push('') // npm rebuild with lifecycle scripts explicitly enabled
|
||||
slots.push('') // chmod prebuilds after rebuild
|
||||
const repairProbe = opts.repairProbe === 'ok' ? 'ORCA-NPTY-PROBE-OK\n' : 'MISSING\n'
|
||||
slots.push(repairProbe)
|
||||
if (!repairProbe.includes('ORCA-NPTY-PROBE-OK')) {
|
||||
slots.push('') // cat stderr after unsuccessful rebuild
|
||||
}
|
||||
slots.push('') // rm -f stderr after rebuild probe
|
||||
}
|
||||
}
|
||||
slots.push('DEAD', 'READY')
|
||||
return slots
|
||||
|
|
@ -207,6 +239,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
// a leaked response. clearAllMocks doesn't drop the queue (it only clears
|
||||
// .mock.calls), so we explicitly mockReset.
|
||||
vi.mocked(execCommand).mockReset()
|
||||
vi.mocked(uploadDirectory).mockResolvedValue(undefined)
|
||||
sftpCapture.paths.length = 0
|
||||
for (const k of Object.keys(sftpCapture.contents)) {
|
||||
delete sftpCapture.contents[k]
|
||||
|
|
@ -256,12 +289,17 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
// break `require('node-pty')`.
|
||||
expect(parsed.type).toBe('commonjs')
|
||||
expect(parsed.dependencies).toEqual({ '@parcel/watcher': '2.5.6', 'node-pty': '1.1.0' })
|
||||
expect(parsed.allowScripts).toEqual({
|
||||
'@parcel/watcher@2.5.6': true,
|
||||
'node-pty@1.1.0': true
|
||||
})
|
||||
|
||||
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
|
||||
const npmInstallIdx = execCalls.findIndex(
|
||||
(c) => c.includes('npm install') && c.includes('node-pty') && c.includes('@parcel/watcher')
|
||||
)
|
||||
expect(npmInstallIdx).toBeGreaterThanOrEqual(0)
|
||||
expect(execCalls[npmInstallIdx]).toContain('--ignore-scripts=false')
|
||||
// Pin actual ordering: number of execCommand calls observed at the moment
|
||||
// ws.end() ran for package.json must be < the index of `npm install`.
|
||||
// Catches a future refactor that fires SFTP-write and npm install via
|
||||
|
|
@ -376,7 +414,96 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
expect(warnMessages.some((m) => m.includes('[ssh-relay][NPTY-MISSING]'))).toBe(true)
|
||||
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('rebuilds unloadable native deps and recovers before first relay launch', async () => {
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed(makeExecResponses({ npmInstall: 'ok', probe: 'missing', repairProbe: 'ok' }))
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
|
||||
const failedProbeIdx = execCalls.findIndex((c) => c.includes('require("node-pty")'))
|
||||
const rebuildIdx = execCalls.findIndex((c) => c.includes('npm rebuild'))
|
||||
const repairedProbeIdx = execCalls.findIndex(
|
||||
(c, index) => index > rebuildIdx && c.includes('require("node-pty")')
|
||||
)
|
||||
expect(rebuildIdx).toBeGreaterThan(failedProbeIdx)
|
||||
expect(execCalls[rebuildIdx]).toContain('--ignore-scripts=false')
|
||||
expect(repairedProbeIdx).toBeGreaterThan(rebuildIdx)
|
||||
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes('[ssh-relay][NPTY-MISSING]'))).toBe(false)
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('propagates an SSH-channel failure from the post-rebuild re-probe', async () => {
|
||||
// Why: the rebuild itself degrades gracefully, but the verification probe
|
||||
// after it must still surface transport death — conflating a dead channel
|
||||
// with "native deps missing" would finalize a half-repaired install.
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64', // uname
|
||||
'/home/u', // $HOME
|
||||
'', // mkdir remoteDir (uploadRelay)
|
||||
'', // chmod +x node
|
||||
'', // npm install native deps
|
||||
'', // chmod prebuilds
|
||||
'MISSING\n', // first probe: require() fails
|
||||
'', // cat probe stderr
|
||||
'', // rm probe stderr
|
||||
'', // npm rebuild native deps
|
||||
'', // chmod prebuilds after rebuild
|
||||
{ reject: 'SSH channel closed during native deps re-probe' } // re-probe rejects
|
||||
])
|
||||
|
||||
await expect(deployAndLaunchRelay(conn)).rejects.toThrow(/SSH channel closed/)
|
||||
|
||||
// Rebuild failure is swallowed; a re-probe transport failure must not be.
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes('[ssh-relay][NPTY-MISSING]'))).toBe(false)
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('aborts an in-progress native install and releases its lock at deploy timeout', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'', // mkdir remoteDir
|
||||
'' // chmod +x node
|
||||
])
|
||||
let installSignal: AbortSignal | undefined
|
||||
vi.mocked(execCommand).mockImplementationOnce((_conn, command, options) => {
|
||||
expect(command).toContain('npm install')
|
||||
installSignal = options?.signal
|
||||
return new Promise<string>((_resolve, reject) => {
|
||||
installSignal?.addEventListener('abort', () => reject(installSignal?.reason), {
|
||||
once: true
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
const promise = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.waitFor(() => expect(installSignal).toBeDefined())
|
||||
|
||||
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
|
||||
|
||||
const result = await promise
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toBe(
|
||||
`Relay deployment timed out after ${RELAY_DEPLOY_TIMEOUT_MS / 1000}s`
|
||||
)
|
||||
expect(installSignal?.aborted).toBe(true)
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('lets a probe SSH-channel failure bubble up rather than silently mapping to MISSING', async () => {
|
||||
|
|
@ -479,9 +606,9 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
expect(chmodPrebuildsIdx).toBeGreaterThan(npmIdx)
|
||||
expect(probeIdx).toBeGreaterThan(chmodPrebuildsIdx)
|
||||
|
||||
// Happy path: finalize exactly once, abandon never.
|
||||
// Hold the install lock through launch, then release it exactly once.
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('matches the sentinel even with bashrc/MOTD noise prefixed to probe stdout', async () => {
|
||||
|
|
@ -522,7 +649,7 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes('[ssh-relay][NPTY-MISSING]'))).toBe(true)
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('keeps Windows node-pty probe failures non-fatal by checking LASTEXITCODE', async () => {
|
||||
|
|
@ -535,7 +662,8 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
'', // mkdir remoteDir
|
||||
'', // npm install native deps
|
||||
'MISSING\n', // native process exit normalized by PowerShell command
|
||||
'', // remove probe stderr file
|
||||
'', // npm rebuild native deps
|
||||
'MISSING\n', // rebuilt native process still cannot load
|
||||
'', // no persisted active pipe marker
|
||||
'WAITING',
|
||||
'', // WMI relay launch
|
||||
|
|
@ -554,11 +682,29 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
const probeScript = decodePowerShellCommand(probeCommand) ?? ''
|
||||
expect(probeScript).toContain('$LASTEXITCODE -ne 0')
|
||||
expect(probeScript).toContain("'MISSING'")
|
||||
expect(probeScript).toContain('loadNativeModule')
|
||||
|
||||
const npmScripts = vi
|
||||
.mocked(execCommand)
|
||||
.mock.calls.map(([, command]) => decodePowerShellCommand(command) ?? '')
|
||||
.filter((script) => script.includes('npm install') || script.includes('npm rebuild'))
|
||||
expect(npmScripts).toHaveLength(2)
|
||||
expect(npmScripts.every((script) => script.includes('--ignore-scripts=false'))).toBe(true)
|
||||
expect(
|
||||
npmScripts.every((script) =>
|
||||
script.includes('if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }')
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
vi
|
||||
.mocked(execCommand)
|
||||
.mock.calls.some(([, command]) => command.includes('.npty-probe.stderr'))
|
||||
).toBe(false)
|
||||
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes('[ssh-relay][NPTY-MISSING]'))).toBe(true)
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('includes the platform tuple in NPTY-MISSING and native install failure logs', async () => {
|
||||
|
|
@ -607,8 +753,8 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'MISSING', // first native-deps probe before lock
|
||||
'MISSING', // re-probe after lock
|
||||
'ORCA-NATIVE-DEPS-MISSING:@parcel/watcher\nMISSING', // first probe before lock
|
||||
'ORCA-NATIVE-DEPS-MISSING:@parcel/watcher\nMISSING', // re-probe after lock
|
||||
'', // npm install native deps
|
||||
'', // chmod prebuilds
|
||||
'ORCA-NPTY-PROBE-OK\n',
|
||||
|
|
@ -619,7 +765,10 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(vi.mocked(acquireInstallLock)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(tryAcquireRelayRepairLock)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(tryAcquireRelayRepairLock).mock.calls[0]?.[3]?.signal).toBeInstanceOf(
|
||||
AbortSignal
|
||||
)
|
||||
expect(vi.mocked(finalizeInstall)).toHaveBeenCalledTimes(1)
|
||||
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
|
||||
expect(
|
||||
|
|
@ -627,6 +776,228 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
(c) => c.includes('npm install') && c.includes('node-pty') && c.includes('@parcel/watcher')
|
||||
)
|
||||
).toBe(true)
|
||||
const installCommand = execCalls.find((c) => c.includes('npm install')) ?? ''
|
||||
expect(installCommand).toContain('node_modules/@parcel/watcher')
|
||||
expect(installCommand).toContain("-name 'watcher-*'")
|
||||
expect(installCommand).not.toContain("rm -rf 'node_modules/node-pty'")
|
||||
})
|
||||
|
||||
it('launches an already-installed relay in degraded mode when repair throws', async () => {
|
||||
// Why: a repair failure on a completed dir (e.g. offline/proxy-locked npm)
|
||||
// must not block the connection — the relay still serves fs/git/preflight.
|
||||
// Pre-fix this rethrew and aborted the whole deploy. The next reconnect
|
||||
// retries, so a transient failure self-heals.
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true)
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'MISSING', // health probe: require() fails
|
||||
'MISSING', // re-probe after lock
|
||||
{ reject: 'npm ERR! network ETIMEDOUT' }, // npm install fails (offline)
|
||||
'DEAD',
|
||||
'READY'
|
||||
])
|
||||
|
||||
// Deploy must resolve (degraded), not reject.
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes('launching degraded'))).toBe(true)
|
||||
})
|
||||
|
||||
it('retains the repair lock when remote command termination is unconfirmed', async () => {
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true)
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
vi.mocked(execCommand)
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/u')
|
||||
.mockResolvedValueOnce('MISSING')
|
||||
.mockResolvedValueOnce('MISSING')
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('npm termination was not confirmed'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
)
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('READY')
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((message) => message.includes('launching degraded'))).toBe(true)
|
||||
})
|
||||
|
||||
it('retains the first-install lock when an aborted npm install has unconfirmed teardown', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'', // mkdir remoteDir
|
||||
'' // chmod +x node
|
||||
])
|
||||
let installSignal: AbortSignal | undefined
|
||||
vi.mocked(execCommand).mockImplementationOnce((_conn, command, options) => {
|
||||
expect(command).toContain('npm install')
|
||||
installSignal = options?.signal
|
||||
return new Promise<string>((_resolve, reject) => {
|
||||
installSignal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
// Mirrors execCommand's bounded close grace when ssh2 never
|
||||
// confirms that the remote npm process stopped.
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
Object.assign(new Error('npm teardown remained unconfirmed'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
),
|
||||
5_000
|
||||
)
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.waitFor(() => expect(installSignal).toBeDefined())
|
||||
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
|
||||
const result = await deploy
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toContain('Relay deployment timed out')
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('does not finalize or release a first-install lock after unconfirmed rebuild teardown', async () => {
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
vi.mocked(execCommand)
|
||||
.mockResolvedValueOnce('__ORCA_REMOTE_PLATFORM__ Linux x86_64')
|
||||
.mockResolvedValueOnce('/home/u')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('MISSING')
|
||||
.mockResolvedValueOnce('rebuild diagnostics')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockRejectedValueOnce(
|
||||
Object.assign(new Error('rebuild termination was not confirmed'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
)
|
||||
|
||||
await expect(deployAndLaunchRelay(conn)).rejects.toThrow(
|
||||
'rebuild termination was not confirmed'
|
||||
)
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('retains the first-install lock when an aborted rebuild has unconfirmed teardown', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'', // mkdir remoteDir
|
||||
'', // chmod +x node
|
||||
'', // npm install
|
||||
'', // chmod prebuilds
|
||||
'MISSING',
|
||||
'rebuild diagnostics',
|
||||
'' // remove probe diagnostics
|
||||
])
|
||||
let rebuildSignal: AbortSignal | undefined
|
||||
vi.mocked(execCommand).mockImplementationOnce((_conn, command, options) => {
|
||||
expect(command).toContain('npm rebuild')
|
||||
rebuildSignal = options?.signal
|
||||
return new Promise<string>((_resolve, reject) => {
|
||||
rebuildSignal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
setTimeout(
|
||||
() =>
|
||||
reject(
|
||||
Object.assign(new Error('rebuild teardown remained unconfirmed'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
),
|
||||
5_000
|
||||
)
|
||||
},
|
||||
{ once: true }
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
const deploy = deployAndLaunchRelay(conn).catch((err: Error) => err)
|
||||
await vi.waitFor(() => expect(rebuildSignal).toBeDefined())
|
||||
await vi.advanceTimersByTimeAsync(RELAY_DEPLOY_TIMEOUT_MS)
|
||||
const result = await deploy
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect((result as Error).message).toContain('Relay deployment timed out')
|
||||
await vi.advanceTimersByTimeAsync(5_000)
|
||||
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it.each(['busy', 'error'] as const)('launches degraded when lock is %s', async (lockResult) => {
|
||||
// Why: lock contention/wedge must not block a completed relay from launching
|
||||
// in degraded mode — repair is best-effort and we hold no lock to release.
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true)
|
||||
vi.mocked(tryAcquireRelayRepairLock).mockResolvedValueOnce(lockResult)
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed(['__ORCA_REMOTE_PLATFORM__ Linux x86_64', '/home/u', 'MISSING', 'DEAD', 'READY'])
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).not.toHaveBeenCalled()
|
||||
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
|
||||
expect(execCalls.some((c) => c.includes('npm install'))).toBe(false)
|
||||
const warnMessages = warnSpy.mock.calls.map((args) => String(args[0] ?? ''))
|
||||
expect(warnMessages.some((m) => m.includes(`repair lock is ${lockResult}`))).toBe(true)
|
||||
})
|
||||
|
||||
it('loads native bindings when checking whether a completed relay needs repair', async () => {
|
||||
vi.mocked(isRelayAlreadyInstalled).mockResolvedValue(true)
|
||||
const conn = makeMockConnection(sftpCapture)
|
||||
feed([
|
||||
'__ORCA_REMOTE_PLATFORM__ Linux x86_64',
|
||||
'/home/u',
|
||||
'ORCA-NATIVE-DEPS-OK',
|
||||
'DEAD',
|
||||
'READY'
|
||||
])
|
||||
|
||||
await deployAndLaunchRelay(conn)
|
||||
|
||||
const healthProbe = vi
|
||||
.mocked(execCommand)
|
||||
.mock.calls.map(([, c]) => c)
|
||||
.find((c) => c.includes('ORCA-NATIVE-DEPS-OK'))
|
||||
expect(healthProbe).toContain('require("node-pty")')
|
||||
expect(healthProbe).toContain('loadNativeModule')
|
||||
expect(healthProbe).toContain('require("@parcel/watcher")')
|
||||
expect(healthProbe).not.toContain('require.resolve')
|
||||
})
|
||||
|
||||
it('does not mutate an existing relay dir when required native deps are present', async () => {
|
||||
|
|
@ -643,7 +1014,9 @@ describe('installNativeDeps (via deployAndLaunchRelay)', () => {
|
|||
await deployAndLaunchRelay(conn)
|
||||
|
||||
expect(vi.mocked(acquireInstallLock)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(tryAcquireRelayRepairLock)).toHaveBeenCalledTimes(1)
|
||||
expect(vi.mocked(finalizeInstall)).not.toHaveBeenCalled()
|
||||
expect(vi.mocked(abandonInstall)).toHaveBeenCalledTimes(1)
|
||||
const execCalls = vi.mocked(execCommand).mock.calls.map(([, c]) => c)
|
||||
expect(execCalls.some((c) => c.includes('npm install'))).toBe(false)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,153 @@
|
|||
import type { SshConnection } from './ssh-connection'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import {
|
||||
acquireInstallLockParentCommand,
|
||||
lockAgeSecondsCommand,
|
||||
probeInstallLockExistsCommand,
|
||||
tryCreateInstallLockCommand,
|
||||
tryStealInstallLockCommand
|
||||
} from './ssh-relay-install-lock-commands'
|
||||
import { isRelayGcClaimed } from './ssh-relay-gc-claim'
|
||||
import {
|
||||
getRemoteHostPlatform,
|
||||
isWindowsRemoteHost,
|
||||
joinRemotePath,
|
||||
type RemoteHostPlatform
|
||||
} from './ssh-remote-platform'
|
||||
import { INSTALL_LOCK_STALE_SECONDS, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock'
|
||||
import { removeRemoteTreeCommand } from './ssh-remote-commands'
|
||||
|
||||
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')
|
||||
|
||||
export type RelayRepairLockResult = 'acquired' | 'busy' | 'gc' | 'error'
|
||||
|
||||
function execHostCommand(
|
||||
conn: SshConnection,
|
||||
host: RemoteHostPlatform,
|
||||
command: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<string> {
|
||||
return execCommand(conn, command, { wrapCommand: !isWindowsRemoteHost(host), signal })
|
||||
}
|
||||
|
||||
/**
|
||||
* Try once to acquire the install lock for best-effort repair work.
|
||||
*
|
||||
* Why: a completed relay can launch in degraded mode, so repair must not wait
|
||||
* behind another installer. An already-stale lock is still recovered now.
|
||||
*/
|
||||
export async function tryAcquireRelayRepairLock(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<RelayRepairLockResult> {
|
||||
const lockDir = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
|
||||
try {
|
||||
const gcClaimedBeforeAcquire = await isRelayGcClaimed(
|
||||
conn,
|
||||
remoteRelayDir,
|
||||
host,
|
||||
options?.signal
|
||||
).catch(() => undefined)
|
||||
options?.signal?.throwIfAborted()
|
||||
if (gcClaimedBeforeAcquire === true) {
|
||||
return 'gc'
|
||||
}
|
||||
if (gcClaimedBeforeAcquire !== false) {
|
||||
return 'error'
|
||||
}
|
||||
await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
acquireInstallLockParentCommand(host, remoteRelayDir),
|
||||
options?.signal
|
||||
)
|
||||
const firstAttempt = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryCreateInstallLockCommand(host, lockDir),
|
||||
options?.signal
|
||||
)
|
||||
if (firstAttempt.trim().endsWith('OK')) {
|
||||
return finishRepairLockAcquire(conn, remoteRelayDir, lockDir, host, options?.signal)
|
||||
}
|
||||
const steal = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
tryStealInstallLockCommand(host, lockDir, INSTALL_LOCK_STALE_SECONDS),
|
||||
options?.signal
|
||||
)
|
||||
if (steal.trim().endsWith('OK')) {
|
||||
console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`)
|
||||
return finishRepairLockAcquire(conn, remoteRelayDir, lockDir, host, options?.signal)
|
||||
}
|
||||
return classifyRepairLockContention(conn, remoteRelayDir, lockDir, host, options?.signal)
|
||||
} catch {
|
||||
options?.signal?.throwIfAborted()
|
||||
return classifyRepairLockContention(conn, remoteRelayDir, lockDir, host, options?.signal)
|
||||
}
|
||||
}
|
||||
|
||||
async function classifyRepairLockContention(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
lockDir: string,
|
||||
host: RemoteHostPlatform,
|
||||
signal?: AbortSignal
|
||||
): Promise<RelayRepairLockResult> {
|
||||
const gcClaimed = await isRelayGcClaimed(conn, remoteRelayDir, host, signal).catch(
|
||||
() => undefined
|
||||
)
|
||||
signal?.throwIfAborted()
|
||||
if (gcClaimed === true) {
|
||||
return 'gc'
|
||||
}
|
||||
if (gcClaimed !== false) {
|
||||
return 'error'
|
||||
}
|
||||
|
||||
const lockProbe = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
probeInstallLockExistsCommand(host, lockDir),
|
||||
signal
|
||||
).catch(() => '')
|
||||
signal?.throwIfAborted()
|
||||
if (lockProbe.trim() !== 'LOCKED') {
|
||||
return 'error'
|
||||
}
|
||||
const ageOutput = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
lockAgeSecondsCommand(host, lockDir),
|
||||
signal
|
||||
).catch(() => '')
|
||||
signal?.throwIfAborted()
|
||||
const ageSeconds = Number.parseInt(ageOutput.trim(), 10)
|
||||
// Why: GC may remove stale locks, so only a positively observed fresh lock
|
||||
// proves that another launch/repair owner is fencing this directory.
|
||||
return Number.isFinite(ageSeconds) && ageSeconds >= 0 && ageSeconds <= INSTALL_LOCK_STALE_SECONDS
|
||||
? 'busy'
|
||||
: 'error'
|
||||
}
|
||||
|
||||
async function finishRepairLockAcquire(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
lockDir: string,
|
||||
host: RemoteHostPlatform,
|
||||
signal?: AbortSignal
|
||||
): Promise<RelayRepairLockResult> {
|
||||
const gcClaimed = await isRelayGcClaimed(conn, remoteRelayDir, host, signal).catch(
|
||||
() => undefined
|
||||
)
|
||||
if (gcClaimed === false && !signal?.aborted) {
|
||||
return 'acquired'
|
||||
}
|
||||
// Why: GC may win its stable sibling claim while this command creates the
|
||||
// in-tree lock. Back out before npm can mutate a directory being renamed.
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {})
|
||||
signal?.throwIfAborted()
|
||||
return gcClaimed ? 'gc' : 'error'
|
||||
}
|
||||
|
|
@ -18,11 +18,19 @@ import {
|
|||
readLocalFullVersion,
|
||||
computeRemoteRelayDir,
|
||||
isRelayAlreadyInstalled,
|
||||
acquireInstallLock,
|
||||
finalizeInstall,
|
||||
abandonInstall,
|
||||
gcOldRelayVersions
|
||||
} from './ssh-relay-versioned-install'
|
||||
import { acquireInstallLock } from './ssh-relay-install-lock'
|
||||
import { tryAcquireRelayRepairLock } from './ssh-relay-repair-lock'
|
||||
import {
|
||||
isRelayGcClaimed,
|
||||
relayGcClaimPath,
|
||||
releaseRelayGcClaim,
|
||||
tryAcquireRelayGcClaim,
|
||||
waitForRelayGcClaimRelease
|
||||
} from './ssh-relay-gc-claim'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
import type { SshConnection } from './ssh-connection'
|
||||
|
|
@ -88,6 +96,18 @@ describe('isRelayAlreadyInstalled', () => {
|
|||
expect(await isRelayAlreadyInstalled(conn, '/r')).toBe(false)
|
||||
})
|
||||
|
||||
it('does not convert an aborted install probe into a missing install', async () => {
|
||||
const controller = new AbortController()
|
||||
mockExec.mockImplementationOnce(async () => {
|
||||
controller.abort()
|
||||
throw Object.assign(new Error('cancelled'), { name: 'AbortError' })
|
||||
})
|
||||
|
||||
await expect(
|
||||
isRelayAlreadyInstalled(conn, '/r', undefined, { signal: controller.signal })
|
||||
).rejects.toMatchObject({ name: 'AbortError' })
|
||||
})
|
||||
|
||||
it('keeps default probe failures as not installed for SSH session-limit-shaped errors', async () => {
|
||||
const sessionLimitError = Object.assign(new Error('(SSH) Channel open failure: open failed'), {
|
||||
reason: 4
|
||||
|
|
@ -124,26 +144,130 @@ describe('acquireInstallLock', () => {
|
|||
})
|
||||
|
||||
it('returns when mkdir reports OK', async () => {
|
||||
// 1st call: mkdir -p remoteRelayDir
|
||||
// 2nd call: mkdir lockDir → OK
|
||||
mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('OK')
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
await acquireInstallLock(conn, '/r')
|
||||
expect(mockExec).toHaveBeenCalledTimes(2)
|
||||
expect(mockExec).toHaveBeenCalledTimes(4)
|
||||
})
|
||||
|
||||
it('recovers a stale sibling GC claim before first-install lock acquisition', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
|
||||
await expect(acquireInstallLock(conn, '/r')).resolves.toBeUndefined()
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands[1]).toContain('lock_tombstone')
|
||||
expect(commands[4]).toBe("mkdir -p '/r'")
|
||||
})
|
||||
|
||||
it('returns immediately without deleting a live repair lock', async () => {
|
||||
// Why: repair can run npm install plus rebuild under the same lock; a
|
||||
// second reconnect must launch degraded rather than corrupt node_modules.
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce('0')
|
||||
|
||||
await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe('busy')
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands.some((command) => command.includes('lock_tombstone'))).toBe(true)
|
||||
expect(commands.filter((command) => command.startsWith('rm -rf'))).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('does not report stale or indeterminate contention as a launch fence', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce(`${21 * 60}`)
|
||||
|
||||
await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe('error')
|
||||
|
||||
mockExec.mockReset().mockRejectedValueOnce(new Error('claim probe failed'))
|
||||
await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe('error')
|
||||
})
|
||||
|
||||
it('recovers a stale best-effort repair lock without polling', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
|
||||
await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe('acquired')
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands.some((command) => command.includes('lock_tombstone'))).toBe(true)
|
||||
expect(commands.filter((command) => command.includes('lock_tombstone'))).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('backs out when GC claims the sibling path during repair lock acquisition', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await expect(tryAcquireRelayRepairLock(conn, '/r')).resolves.toBe('gc')
|
||||
|
||||
const lastCommand = mockExec.mock.calls.at(-1)?.[1] ?? ''
|
||||
expect(lastCommand).toBe("rm -rf '/r/.install-lock'")
|
||||
})
|
||||
|
||||
it('propagates cancellation through best-effort repair lock commands', async () => {
|
||||
const abortController = new AbortController()
|
||||
const abortError = Object.assign(new Error('cancelled'), { name: 'AbortError' })
|
||||
mockExec.mockRejectedValueOnce(abortError)
|
||||
|
||||
const promise = tryAcquireRelayRepairLock(conn, '/r', undefined, {
|
||||
signal: abortController.signal
|
||||
})
|
||||
abortController.abort()
|
||||
|
||||
await expect(promise).rejects.toMatchObject({ name: 'AbortError' })
|
||||
expect(mockExec.mock.calls[0]?.[2]?.signal).toBe(abortController.signal)
|
||||
})
|
||||
|
||||
it('polls until the lock becomes available (concurrent installer wins, then we acquire)', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
// Sequence:
|
||||
// 1. mkdir -p (parent dir prep)
|
||||
// 2. mkdir lockDir → BUSY (someone else holds it)
|
||||
// 3. mkdir lockDir → BUSY again
|
||||
// 4. mkdir lockDir → OK (concurrent installer released)
|
||||
mockExec
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('OK')
|
||||
let createAttempts = 0
|
||||
mockExec.mockImplementation(async (_conn: unknown, command: string) => {
|
||||
if (command.includes('.gc-claim')) {
|
||||
return 'OPEN'
|
||||
}
|
||||
if (command.startsWith('mkdir -p')) {
|
||||
return ''
|
||||
}
|
||||
if (command.includes('lock_tombstone')) {
|
||||
return 'BUSY'
|
||||
}
|
||||
if (command.includes('.install-lock')) {
|
||||
createAttempts++
|
||||
return createAttempts >= 3 ? 'OK' : 'BUSY'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const promise = acquireInstallLock(conn, '/r')
|
||||
// Drive the polling loop: each iteration awaits a 1s timer.
|
||||
|
|
@ -159,36 +283,48 @@ describe('acquireInstallLock', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('steals a stale lock and retries with a reset timeout window', async () => {
|
||||
it('immediately tries to steal an already-stale lock', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('BUSY')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
|
||||
await expect(acquireInstallLock(conn, '/r')).resolves.toBeUndefined()
|
||||
|
||||
const cmds = mockExec.mock.calls.map(([, c]) => c)
|
||||
expect(cmds).toHaveLength(5)
|
||||
expect(cmds[3]).toContain('lock_tombstone')
|
||||
})
|
||||
|
||||
it('retries stale takeover when a fresh lock ages out during the wait', async () => {
|
||||
vi.useFakeTimers({ now: 1_700_000_000_000 })
|
||||
try {
|
||||
let mkdirCalls = 0
|
||||
const recoverableAt = Date.now() + 60_000
|
||||
mockExec.mockImplementation(async (_conn: unknown, cmd: string) => {
|
||||
if (cmd.includes('.gc-claim')) {
|
||||
return 'OPEN'
|
||||
}
|
||||
if (cmd.startsWith('mkdir -p')) {
|
||||
return ''
|
||||
}
|
||||
if (cmd.includes('lock_tombstone')) {
|
||||
return Date.now() >= recoverableAt ? 'OK' : 'BUSY'
|
||||
}
|
||||
if (cmd.includes('mkdir') && cmd.includes('.install-lock')) {
|
||||
mkdirCalls++
|
||||
return mkdirCalls > 200 ? 'OK' : 'BUSY'
|
||||
}
|
||||
if (cmd.includes('stat')) {
|
||||
return `${Math.floor((Date.now() - 10 * 60 * 1000) / 1000)}\n`
|
||||
}
|
||||
if (cmd.startsWith('rm -rf')) {
|
||||
mkdirCalls = 1000
|
||||
return ''
|
||||
return 'BUSY'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const promise = acquireInstallLock(conn, '/r')
|
||||
// Drive through the full timeout (120s) so the stale-recovery branch
|
||||
// fires, then drive a few more seconds for the post-recovery retry.
|
||||
await vi.advanceTimersByTimeAsync(125_000)
|
||||
await promise
|
||||
await vi.advanceTimersByTimeAsync(61_000)
|
||||
|
||||
const cmds = mockExec.mock.calls.map(([, c]) => c)
|
||||
expect(cmds.some((c) => c.includes('rm -rf') && c.includes('.install-lock'))).toBe(true)
|
||||
await expect(promise).resolves.toBeUndefined()
|
||||
expect(mockExec.mock.calls.filter(([, cmd]) => cmd.includes('lock_tombstone'))).toHaveLength(
|
||||
2
|
||||
)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
|
|
@ -198,26 +334,86 @@ describe('acquireInstallLock', () => {
|
|||
vi.useFakeTimers({ now: 1_700_000_000_000 })
|
||||
try {
|
||||
mockExec.mockImplementation(async (_conn: unknown, cmd: string) => {
|
||||
if (cmd.includes('.gc-claim')) {
|
||||
return 'OPEN'
|
||||
}
|
||||
if (cmd.startsWith('mkdir -p')) {
|
||||
return ''
|
||||
}
|
||||
if (cmd.includes('lock_tombstone')) {
|
||||
return 'BUSY'
|
||||
}
|
||||
if (cmd.includes('mkdir') && cmd.includes('.install-lock')) {
|
||||
return 'BUSY'
|
||||
}
|
||||
if (cmd.includes('stat')) {
|
||||
return `${Math.floor(Date.now() / 1000)}\n`
|
||||
return '0\n'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const rejection = expect(acquireInstallLock(conn, '/r')).rejects.toThrow(/not yet stale/i)
|
||||
await vi.advanceTimersByTimeAsync(125_000)
|
||||
const rejection = expect(acquireInstallLock(conn, '/r')).rejects.toThrow(
|
||||
/another install is still in progress/i
|
||||
)
|
||||
await vi.advanceTimersByTimeAsync(905_000)
|
||||
await rejection
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('stops polling when the caller aborts the lock wait', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const abortController = new AbortController()
|
||||
mockExec.mockImplementation(async (_conn: unknown, cmd: string) =>
|
||||
cmd.includes('.gc-claim') ? 'OPEN' : 'BUSY'
|
||||
)
|
||||
|
||||
const promise = acquireInstallLock(conn, '/r', undefined, {
|
||||
signal: abortController.signal
|
||||
})
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
abortController.abort()
|
||||
await expect(promise).rejects.toMatchObject({ name: 'AbortError' })
|
||||
|
||||
const callCountAfterAbort = mockExec.mock.calls.length
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(mockExec).toHaveBeenCalledTimes(callCountAfterAbort)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps waiting while a slow first installer is within the deploy bound', async () => {
|
||||
vi.useFakeTimers({ now: 1_700_000_000_000 })
|
||||
try {
|
||||
const availableAt = Date.now() + 500_000
|
||||
mockExec.mockImplementation(async (_conn: unknown, cmd: string) => {
|
||||
if (cmd.includes('.gc-claim')) {
|
||||
return 'OPEN'
|
||||
}
|
||||
if (cmd.startsWith('mkdir -p')) {
|
||||
return ''
|
||||
}
|
||||
if (cmd.includes('mkdir') && cmd.includes('.install-lock')) {
|
||||
return Date.now() >= availableAt ? 'OK' : 'BUSY'
|
||||
}
|
||||
return ''
|
||||
})
|
||||
|
||||
const promise = acquireInstallLock(conn, '/r')
|
||||
await vi.advanceTimersByTimeAsync(501_000)
|
||||
|
||||
await expect(promise).resolves.toBeUndefined()
|
||||
expect(
|
||||
mockExec.mock.calls.filter(([, cmd]) => cmd.includes('lock_tombstone')).length
|
||||
).toBeGreaterThan(1)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('finalizeInstall writes .install-complete then removes the lock', async () => {
|
||||
mockExec.mockResolvedValueOnce('').mockResolvedValueOnce('')
|
||||
await finalizeInstall(conn, '/r')
|
||||
|
|
@ -238,26 +434,220 @@ describe('acquireInstallLock', () => {
|
|||
})
|
||||
})
|
||||
|
||||
describe('relay GC claim', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockExec.mockReset()
|
||||
})
|
||||
|
||||
it('parses explicit claim markers through remote startup noise', async () => {
|
||||
mockExec.mockResolvedValueOnce('Welcome to host\nLOCKED\n')
|
||||
await expect(isRelayGcClaimed(conn, '/relay/version')).resolves.toBe(true)
|
||||
|
||||
mockExec.mockResolvedValueOnce('Last login: today\nOPEN\n')
|
||||
await expect(isRelayGcClaimed(conn, '/relay/version')).resolves.toBe(false)
|
||||
})
|
||||
|
||||
it('rejects missing or conflicting claim markers', async () => {
|
||||
mockExec.mockResolvedValueOnce('Welcome to host\n')
|
||||
await expect(isRelayGcClaimed(conn, '/relay/version')).rejects.toThrow('Inconclusive')
|
||||
|
||||
mockExec.mockResolvedValueOnce('LOCKED\nOPEN\n')
|
||||
await expect(isRelayGcClaimed(conn, '/relay/version')).rejects.toThrow('Inconclusive')
|
||||
})
|
||||
|
||||
it('uses a stable sibling path outside the recursively deleted install', async () => {
|
||||
mockExec.mockResolvedValueOnce('OK').mockResolvedValueOnce('')
|
||||
|
||||
await expect(tryAcquireRelayGcClaim(conn, '/relay/version')).resolves.toEqual(
|
||||
expect.any(String)
|
||||
)
|
||||
|
||||
expect(relayGcClaimPath('/relay/version')).toBe('/relay/version.gc-claim')
|
||||
expect(mockExec.mock.calls[0]?.[1]).toContain("mkdir '/relay/version.gc-claim'")
|
||||
})
|
||||
|
||||
it('recovers and removes a stale sibling claim before retrying deploy', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
|
||||
await waitForRelayGcClaimRelease(conn, '/relay/version')
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands[1]).toContain('lock_tombstone')
|
||||
expect(commands[3]).toContain("rm -rf '/relay/version.gc-claim'")
|
||||
})
|
||||
|
||||
it('keeps waiting after losing a recovered claim until the claim is actually gone', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('LOCKED')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('LOST')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
|
||||
await waitForRelayGcClaimRelease(conn, '/relay/version')
|
||||
|
||||
expect(mockExec).toHaveBeenCalledTimes(5)
|
||||
})
|
||||
|
||||
it('writes and conditionally releases the Windows sibling claim owner token', async () => {
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
mockExec.mockResolvedValueOnce('OK').mockResolvedValueOnce('').mockResolvedValueOnce('RELEASED')
|
||||
|
||||
const token = await tryAcquireRelayGcClaim(conn, 'C:/relay/version', windows)
|
||||
expect(token).toEqual(expect.any(String))
|
||||
await expect(releaseRelayGcClaim(conn, 'C:/relay/version', token!, windows)).resolves.toBe(
|
||||
'released'
|
||||
)
|
||||
|
||||
const ownerScript = decodePowerShellCommand(mockExec.mock.calls[1]?.[1] ?? '')
|
||||
const releaseScript = decodePowerShellCommand(mockExec.mock.calls[2]?.[1] ?? '')
|
||||
expect(ownerScript).toContain('Set-Content -LiteralPath')
|
||||
expect(ownerScript).toContain('.gc-claim/.gc-owner')
|
||||
expect(releaseScript).toContain('Get-Content -LiteralPath')
|
||||
expect(releaseScript).toContain('-cne')
|
||||
expect(releaseScript).toContain('Remove-Item -LiteralPath')
|
||||
expect(releaseScript).toContain("'RELEASED'")
|
||||
expect(releaseScript).toContain("'LOST'")
|
||||
expect(releaseScript).toContain("'UNKNOWN'")
|
||||
expect(releaseScript).not.toContain('}; elseif')
|
||||
expect(releaseScript).not.toContain('{;')
|
||||
})
|
||||
|
||||
it('conditionally releases a claim when the owner write reply is lost', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockRejectedValueOnce(new Error('owner write reply lost'))
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
|
||||
await expect(tryAcquireRelayGcClaim(conn, '/relay/version')).resolves.toBeNull()
|
||||
|
||||
const ownerCommand = mockExec.mock.calls[1]?.[1] ?? ''
|
||||
const releaseCommand = mockExec.mock.calls[2]?.[1] ?? ''
|
||||
const token = ownerCommand.match(/printf %s '([^']+)'/)?.[1]
|
||||
expect(token).toBeTruthy()
|
||||
expect(releaseCommand).toContain(`!= '${token}'`)
|
||||
})
|
||||
})
|
||||
|
||||
describe('gcOldRelayVersions', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks()
|
||||
mockExec.mockReset()
|
||||
})
|
||||
|
||||
it('removes a sibling that is complete, unlocked, and has no live socket', async () => {
|
||||
// ls listing
|
||||
mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.1.0+bbb\n')
|
||||
// For sibling "aaa": LOCKED probe → OPEN, COMPLETE probe → COMPLETE, sock probe → empty (no ALIVE), then rm -rf
|
||||
// For sibling "aaa": safety probes pass, then GC claims the install lock before removal.
|
||||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockResolvedValueOnce('MOVED')
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const lastCmd = mockExec.mock.calls.at(-1)?.[1] ?? ''
|
||||
expect(lastCmd).toContain('rm -rf')
|
||||
expect(lastCmd).toContain('relay-0.1.0+aaa')
|
||||
expect(lastCmd).toContain('relay-0.1.0+aaa.gc-tombstone')
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(
|
||||
commands.some((command) => command === "rm -rf '/home/u/.orca-remote/relay-0.1.0+aaa'")
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('cleans only strict POSIX orphan tombstones even with no relay candidates', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce(
|
||||
[
|
||||
'relay-0.1.0+abc.gc-tombstone.123.456',
|
||||
'relay-0.1.0+abc.gc-tombstone.bad.456',
|
||||
'xrelay-0.1.0+abc.gc-tombstone.123.456',
|
||||
'relay-0.1.0+abc.gc-tombstone.123.456.extra',
|
||||
'relay-0.1.0+abc.gc-tombstone.123.456/child',
|
||||
'logs'
|
||||
].join('\n')
|
||||
)
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const removeCommands = mockExec.mock.calls
|
||||
.map(([, command]) => command)
|
||||
.filter((command) => command.startsWith('rm -rf'))
|
||||
expect(removeCommands).toEqual([
|
||||
"rm -rf '/home/u/.orca-remote/relay-0.1.0+abc.gc-tombstone.123.456'"
|
||||
])
|
||||
})
|
||||
|
||||
it('cleans only strict Windows orphan tombstones even with no relay candidates', async () => {
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
mockExec
|
||||
.mockResolvedValueOnce(
|
||||
'relay-v0.1.0.gc-tombstone.123.456\nrelay-v0.1.0.gc-tombstone.latest.456\n'
|
||||
)
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(conn, 'C:/Users/u', 'C:/Users/u/.orca-remote/relay-0.1.0+bbb', windows)
|
||||
|
||||
const removeScript = decodePowerShellCommand(mockExec.mock.calls[1]?.[1] ?? '')
|
||||
expect(removeScript).toContain('relay-v0.1.0.gc-tombstone.123.456')
|
||||
expect(removeScript).not.toContain('relay-v0.1.0.gc-tombstone.latest.456')
|
||||
})
|
||||
|
||||
it('retries orphan tombstone cleanup on a later GC pass', async () => {
|
||||
const tombstone = 'relay-0.1.0+abc.gc-tombstone.123.456'
|
||||
mockExec
|
||||
.mockResolvedValueOnce(tombstone)
|
||||
.mockRejectedValueOnce(new Error('remove failed'))
|
||||
.mockResolvedValueOnce(tombstone)
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const removeCommands = mockExec.mock.calls
|
||||
.map(([, command]) => command)
|
||||
.filter((command) => command.startsWith('rm -rf'))
|
||||
expect(removeCommands).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('retries only an unknown claim release and stops after observing a lost generation', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockResolvedValueOnce('MOVED')
|
||||
.mockResolvedValueOnce('UNKNOWN')
|
||||
.mockResolvedValueOnce('LOST')
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const releaseCommands = mockExec.mock.calls
|
||||
.map(([, command]) => command)
|
||||
.filter((command) => command.includes('.gc-owner') && command.includes('echo RELEASED'))
|
||||
expect(releaseCommands).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('skips siblings that are missing .install-complete (mid-install or partial)', async () => {
|
||||
|
|
@ -273,8 +663,8 @@ describe('gcOldRelayVersions', () => {
|
|||
it('skips siblings whose .install-lock is held and fresh', async () => {
|
||||
mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
mockExec.mockResolvedValueOnce('LOCKED')
|
||||
// isLockStale: mtime ~now → not stale.
|
||||
mockExec.mockResolvedValueOnce(`${Math.floor(Date.now() / 1000)}\n`)
|
||||
// isLockStale: age ~now → not stale.
|
||||
mockExec.mockResolvedValueOnce('0\n')
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
const cmds = mockExec.mock.calls.map(([, c]) => c)
|
||||
expect(cmds.some((c) => c.includes('rm -rf'))).toBe(false)
|
||||
|
|
@ -283,12 +673,20 @@ describe('gcOldRelayVersions', () => {
|
|||
it('removes a sibling with a stale lock + .install-complete (rm-lock failed mid-finalize)', async () => {
|
||||
mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
mockExec.mockResolvedValueOnce('LOCKED')
|
||||
// isLockStale: mtime well in the past → stale.
|
||||
const staleSec = Math.floor((Date.now() - 10 * 60 * 1000) / 1000)
|
||||
mockExec.mockResolvedValueOnce(`${staleSec}\n`)
|
||||
// isLockStale: age well above the stale window → stale.
|
||||
mockExec.mockResolvedValueOnce(`${21 * 60}\n`)
|
||||
mockExec.mockResolvedValueOnce('COMPLETE') // .install-complete present
|
||||
mockExec.mockResolvedValueOnce('') // socket probe → no ALIVE
|
||||
mockExec.mockResolvedValueOnce('') // rm -rf
|
||||
mockExec.mockResolvedValueOnce('DEAD') // socket probe
|
||||
mockExec.mockResolvedValueOnce('OK') // stable sibling GC claim
|
||||
mockExec.mockResolvedValueOnce('') // write claim ownership token
|
||||
mockExec.mockResolvedValueOnce('LOCKED')
|
||||
mockExec.mockResolvedValueOnce(`${21 * 60}\n`)
|
||||
mockExec.mockResolvedValueOnce('COMPLETE')
|
||||
mockExec.mockResolvedValueOnce('DEAD')
|
||||
mockExec.mockResolvedValueOnce('OWNED')
|
||||
mockExec.mockResolvedValueOnce('MOVED')
|
||||
mockExec.mockResolvedValueOnce('RELEASED') // release sibling claim
|
||||
mockExec.mockResolvedValueOnce('') // remove tombstone
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
const lastCmd = mockExec.mock.calls.at(-1)?.[1] ?? ''
|
||||
expect(lastCmd).toContain('rm -rf')
|
||||
|
|
@ -298,8 +696,15 @@ describe('gcOldRelayVersions', () => {
|
|||
it('GCs a legacy relay-v0.1.0 dir whose daemon is dead (no .install-complete required)', async () => {
|
||||
mockExec.mockResolvedValueOnce('relay-v0.1.0\n')
|
||||
mockExec.mockResolvedValueOnce('OPEN') // not locked
|
||||
mockExec.mockResolvedValueOnce('') // socket probe → no ALIVE (no completeProbe — legacy)
|
||||
mockExec.mockResolvedValueOnce('') // rm -rf
|
||||
mockExec.mockResolvedValueOnce('DEAD') // socket probe (no completeProbe — legacy)
|
||||
mockExec.mockResolvedValueOnce('OK') // GC claims candidate
|
||||
mockExec.mockResolvedValueOnce('')
|
||||
mockExec.mockResolvedValueOnce('OPEN')
|
||||
mockExec.mockResolvedValueOnce('DEAD')
|
||||
mockExec.mockResolvedValueOnce('OWNED')
|
||||
mockExec.mockResolvedValueOnce('MOVED')
|
||||
mockExec.mockResolvedValueOnce('RELEASED')
|
||||
mockExec.mockResolvedValueOnce('')
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
const cmds = mockExec.mock.calls.map(([, c]) => c)
|
||||
expect(cmds.some((c) => c.includes('rm -rf') && c.includes('relay-v0.1.0'))).toBe(true)
|
||||
|
|
@ -327,6 +732,32 @@ describe('gcOldRelayVersions', () => {
|
|||
expect(cmds.some((c) => c.includes('rm -rf'))).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps a sibling when the install-lock probe fails or returns unexpected output', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.1.0+ccc\n')
|
||||
.mockRejectedValueOnce(new Error('lock probe failed'))
|
||||
.mockResolvedValueOnce('INCONCLUSIVE')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
expect(mockExec).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('keeps a sibling when the relay-liveness probe fails or returns unexpected output', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\nrelay-0.1.0+ccc\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockRejectedValueOnce(new Error('liveness probe failed'))
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('INCONCLUSIVE')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
expect(mockExec).toHaveBeenCalledTimes(7)
|
||||
})
|
||||
|
||||
it('probes Windows GC liveness by connecting to named pipes, not process command lines', async () => {
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
mockExec.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
|
|
@ -334,6 +765,14 @@ describe('gcOldRelayVersions', () => {
|
|||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('WAITING')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('WAITING')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockResolvedValueOnce('MOVED')
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
.mockResolvedValueOnce('')
|
||||
|
||||
await gcOldRelayVersions(
|
||||
|
|
@ -366,14 +805,113 @@ describe('gcOldRelayVersions', () => {
|
|||
mockExec
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockResolvedValueOnce('MOVED')
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
.mockResolvedValueOnce('')
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
const cmds = mockExec.mock.calls.map(([, c]) => c)
|
||||
const rmCmds = cmds.filter((c) => c.includes('rm -rf'))
|
||||
const rmCmds = cmds.filter((c) => c.startsWith('rm') && c.includes('gc-tombstone'))
|
||||
expect(rmCmds).toHaveLength(1)
|
||||
expect(rmCmds[0]).toContain('relay-0.1.0+aaa')
|
||||
expect(rmCmds[0]).not.toContain('logs')
|
||||
expect(rmCmds[0]).not.toContain('backup')
|
||||
})
|
||||
|
||||
it('does not remove a candidate claimed by repair after the initial lock probe', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD') // socket probe
|
||||
.mockResolvedValueOnce('OK') // GC sibling claim
|
||||
.mockResolvedValueOnce('') // write claim ownership token
|
||||
.mockResolvedValueOnce('LOCKED') // repair won before the safety recheck
|
||||
.mockResolvedValueOnce('0')
|
||||
.mockResolvedValueOnce('RELEASED') // release GC sibling claim
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(
|
||||
commands.some(
|
||||
(command) =>
|
||||
command.startsWith("rm -rf '/home/u/.orca-remote/relay-0.1.0+aaa'") &&
|
||||
!command.includes('.install-lock')
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('does not move a candidate after losing the sibling claim generation', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('LOST')
|
||||
.mockResolvedValueOnce('LOST')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const commands = mockExec.mock.calls.map(([, command]) => command)
|
||||
expect(commands.some((command) => command.startsWith('mv '))).toBe(false)
|
||||
})
|
||||
|
||||
it('releases the GC claim after a confirmed move failure', async () => {
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockRejectedValueOnce(new Error('move failed'))
|
||||
.mockResolvedValueOnce('RELEASED')
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const lastCommand = mockExec.mock.calls.at(-1)?.[1] ?? ''
|
||||
expect(lastCommand).toContain('rm -rf')
|
||||
expect(lastCommand).toContain('relay-0.1.0+aaa.gc-claim')
|
||||
})
|
||||
|
||||
it('keeps the GC claim when remote move termination is unconfirmed', async () => {
|
||||
const unconfirmed = Object.assign(new Error('move timed out'), {
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
mockExec
|
||||
.mockResolvedValueOnce('relay-0.1.0+aaa\n')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OK')
|
||||
.mockResolvedValueOnce('')
|
||||
.mockResolvedValueOnce('OPEN')
|
||||
.mockResolvedValueOnce('COMPLETE')
|
||||
.mockResolvedValueOnce('DEAD')
|
||||
.mockResolvedValueOnce('OWNED')
|
||||
.mockRejectedValueOnce(unconfirmed)
|
||||
|
||||
await gcOldRelayVersions(conn, '/home/u', '/home/u/.orca-remote/relay-0.1.0+bbb')
|
||||
|
||||
const releaseCommands = mockExec.mock.calls
|
||||
.map(([, command]) => command)
|
||||
.filter((command) => command.includes('relay-0.1.0+aaa.gc-claim') && command.startsWith('rm'))
|
||||
expect(releaseCommands).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -15,16 +15,21 @@ import { existsSync, readFileSync } from 'node:fs'
|
|||
import type { SshConnection } from './ssh-connection'
|
||||
import { RELAY_REMOTE_DIR } from './relay-protocol'
|
||||
import { execCommand } from './ssh-relay-deploy-helpers'
|
||||
import { probeInstallLockExistsCommand } from './ssh-relay-install-lock-commands'
|
||||
import { isRelayInstallLockStale, RELAY_INSTALL_LOCK_NAME } from './ssh-relay-install-lock'
|
||||
import {
|
||||
isRelayGcClaimOwned,
|
||||
releaseRelayGcClaimWithRetry,
|
||||
tryAcquireRelayGcClaim
|
||||
} from './ssh-relay-gc-claim'
|
||||
import { cleanupRelayGcTombstones } from './ssh-relay-gc-tombstone'
|
||||
import {
|
||||
acquireInstallLockParentCommand,
|
||||
listRelayBaseDirsCommand,
|
||||
lockMtimeEpochCommand,
|
||||
probeDirectoryExistsCommand,
|
||||
moveRemoteTreeCommand,
|
||||
probeFileExistsCommand,
|
||||
probeRelayInstalledCommand,
|
||||
relayLivenessProbeCommand,
|
||||
removeRemoteTreeCommand,
|
||||
tryCreateInstallLockCommand,
|
||||
writeRemoteEmptyFileCommand
|
||||
} from './ssh-remote-commands'
|
||||
import {
|
||||
|
|
@ -36,6 +41,7 @@ import {
|
|||
type RemotePathFlavor
|
||||
} from './ssh-remote-platform'
|
||||
import { windowsRelayPipePathsForSocketName } from './ssh-relay-endpoints'
|
||||
import { isUnconfirmedSshCommandTermination } from './ssh-relay-exec-command'
|
||||
import { isSshSessionLimitError } from './ssh-session-limit-error'
|
||||
|
||||
// Why: the GC pass and the version-dir parser must agree on what counts as a
|
||||
|
|
@ -51,18 +57,7 @@ const RELAY_VERSION_DIR_REGEX = /^relay-(v?\d+\.\d+\.\d+(\+[0-9a-f]+)?)$/
|
|||
// living on remote disks forever.
|
||||
const LEGACY_RELAY_DIR_REGEX = /^relay-v\d+\.\d+\.\d+$/
|
||||
|
||||
const INSTALL_LOCK_NAME = '.install-lock'
|
||||
const INSTALL_COMPLETE_NAME = '.install-complete'
|
||||
|
||||
const INSTALL_LOCK_POLL_MS = 1_000
|
||||
const INSTALL_LOCK_TIMEOUT_MS = 120_000
|
||||
// Why: a stale lock dir from a crashed installer must be recoverable without
|
||||
// user intervention. After the timeout we check the lock's mtime; if it's
|
||||
// older than this window the previous installer is assumed dead and we steal
|
||||
// the lock. 2 minutes is well above a normal `npm install node-pty` runtime
|
||||
// (10–60s on slow hosts) so a slow concurrent installer is not falsely
|
||||
// declared dead.
|
||||
const INSTALL_LOCK_STALE_MS = 120_000
|
||||
const DEFAULT_REMOTE_HOST = getRemoteHostPlatform('linux-x64')
|
||||
|
||||
type RelayInstalledProbeOptions = {
|
||||
|
|
@ -145,6 +140,7 @@ export async function isRelayAlreadyInstalled(
|
|||
)
|
||||
return probe.trim() === 'OK'
|
||||
} catch (err) {
|
||||
options?.signal?.throwIfAborted()
|
||||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
|
|
@ -153,101 +149,27 @@ export async function isRelayAlreadyInstalled(
|
|||
}
|
||||
|
||||
/**
|
||||
* Acquire the per-version install lock via atomic `mkdir`. Returns when the
|
||||
* caller owns the lock; throws if the lock could not be acquired within
|
||||
* INSTALL_LOCK_TIMEOUT_MS even after one stale-lock recovery attempt.
|
||||
*
|
||||
* Why mkdir: POSIX `mkdir` is atomic and fails with EEXIST if the dir already
|
||||
* exists, giving us a free mutex. A second concurrent caller polls and
|
||||
* eventually either acquires the lock or steals it after the stale window.
|
||||
*/
|
||||
export async function acquireInstallLock(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<void> {
|
||||
const lockDir = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME)
|
||||
// Why: the parent dir may not exist yet on a first install. mkdir -p is
|
||||
// safe to run multiple times — it's a no-op if the dir already exists.
|
||||
await execHostCommand(conn, host, acquireInstallLockParentCommand(host, remoteRelayDir))
|
||||
|
||||
let start = Date.now()
|
||||
let recoveredOnce = false
|
||||
while (true) {
|
||||
try {
|
||||
const result = await execHostCommand(conn, host, tryCreateInstallLockCommand(host, lockDir))
|
||||
if (result.trim().endsWith('OK')) {
|
||||
return
|
||||
}
|
||||
} catch {
|
||||
/* mkdir failed with non-zero — fall through to BUSY treatment */
|
||||
}
|
||||
if (Date.now() - start >= INSTALL_LOCK_TIMEOUT_MS) {
|
||||
if (recoveredOnce) {
|
||||
throw new Error(
|
||||
`Could not acquire relay install lock at ${lockDir} after ${
|
||||
INSTALL_LOCK_TIMEOUT_MS / 1000
|
||||
}s; another install is in progress or the lock is wedged.`
|
||||
)
|
||||
}
|
||||
// Stale-lock recovery: if the lock dir's mtime is older than the stale
|
||||
// window, the previous installer crashed. Steal it and retry once,
|
||||
// resetting the timeout window so a single post-recovery race doesn't
|
||||
// immediately exhaust the budget.
|
||||
const ageOk = await isLockStale(conn, lockDir, host)
|
||||
if (ageOk) {
|
||||
console.warn(`[ssh-relay] Stealing stale install lock at ${lockDir}`)
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lockDir)).catch(() => {})
|
||||
recoveredOnce = true
|
||||
start = Date.now()
|
||||
continue
|
||||
}
|
||||
throw new Error(
|
||||
`Could not acquire relay install lock at ${lockDir} after ${
|
||||
INSTALL_LOCK_TIMEOUT_MS / 1000
|
||||
}s and the lock is not yet stale.`
|
||||
)
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, INSTALL_LOCK_POLL_MS))
|
||||
}
|
||||
}
|
||||
|
||||
async function isLockStale(
|
||||
conn: SshConnection,
|
||||
lockDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<boolean> {
|
||||
try {
|
||||
// Why: `stat` flags differ between GNU coreutils (Linux) and BSD (macOS).
|
||||
// We try GNU first, then BSD; both produce a Unix epoch in seconds on
|
||||
// stdout. If both fail we conservatively treat the lock as not stale.
|
||||
const out = await execHostCommand(conn, host, lockMtimeEpochCommand(host, lockDir))
|
||||
const mtimeSec = Number.parseInt(out.trim(), 10)
|
||||
if (!Number.isFinite(mtimeSec)) {
|
||||
return false
|
||||
}
|
||||
const ageMs = Date.now() - mtimeSec * 1000
|
||||
return ageMs > INSTALL_LOCK_STALE_MS
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark the install as complete and release the lock. Sentinel ordering is:
|
||||
* write `.install-complete` FIRST, then remove `.install-lock`. This ensures
|
||||
* a sibling dir is never observed by GC as "complete but locked", which
|
||||
* would lead GC to skip a recoverable dir indefinitely.
|
||||
* Mark the install as complete, then normally release the lock. Deploy keeps
|
||||
* the lock through first launch so cross-version GC cannot move the directory
|
||||
* between finalization and daemon liveness becoming observable.
|
||||
*/
|
||||
export async function finalizeInstall(
|
||||
conn: SshConnection,
|
||||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST,
|
||||
options?: { signal?: AbortSignal; releaseLock?: boolean }
|
||||
): Promise<void> {
|
||||
const sentinel = joinRemotePath(host, remoteRelayDir, INSTALL_COMPLETE_NAME)
|
||||
const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME)
|
||||
await execHostCommand(conn, host, writeRemoteEmptyFileCommand(host, sentinel))
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {})
|
||||
const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
|
||||
await execHostCommand(conn, host, writeRemoteEmptyFileCommand(host, sentinel), {
|
||||
signal: options?.signal
|
||||
})
|
||||
if (options?.releaseLock !== false) {
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock), {
|
||||
signal: options?.signal
|
||||
}).catch(() => {})
|
||||
}
|
||||
options?.signal?.throwIfAborted()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -260,7 +182,7 @@ export async function abandonInstall(
|
|||
remoteRelayDir: string,
|
||||
host: RemoteHostPlatform = DEFAULT_REMOTE_HOST
|
||||
): Promise<void> {
|
||||
const lock = joinRemotePath(host, remoteRelayDir, INSTALL_LOCK_NAME)
|
||||
const lock = joinRemotePath(host, remoteRelayDir, RELAY_INSTALL_LOCK_NAME)
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, lock)).catch(() => {})
|
||||
}
|
||||
|
||||
|
|
@ -295,10 +217,14 @@ export async function gcOldRelayVersions(
|
|||
} catch {
|
||||
return
|
||||
}
|
||||
const candidates = listing
|
||||
const entries = listing
|
||||
.split('\n')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
|
||||
await cleanupRelayGcTombstones(conn, baseDir, entries, host)
|
||||
|
||||
const candidates = entries
|
||||
.filter((name) => RELAY_VERSION_DIR_REGEX.test(name))
|
||||
.filter((name) => name !== currentDirName)
|
||||
|
||||
|
|
@ -316,7 +242,47 @@ export async function gcOldRelayVersions(
|
|||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, dir))
|
||||
// Why: the claim is a sibling, so it survives moving/deleting the
|
||||
// candidate and lets installers back out before mutating the old path.
|
||||
const gcClaimToken = await tryAcquireRelayGcClaim(conn, dir, host)
|
||||
if (!gcClaimToken) {
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
let preserveGcClaim = false
|
||||
let gcClaimReleaseNeeded = true
|
||||
try {
|
||||
// Recheck under the stable claim. New installers probe the claim both
|
||||
// before and after creating their in-tree lock, closing both orders.
|
||||
if (!(await isCandidateSafeToRemove(conn, dir, name, host, options))) {
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
if (!(await isRelayGcClaimOwned(conn, dir, gcClaimToken, host))) {
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
const tombstone = `${dir}.gc-tombstone.${process.pid}.${Date.now()}`
|
||||
const moved = await execHostCommand(conn, host, moveRemoteTreeCommand(host, dir, tombstone))
|
||||
if (moved.trim() !== 'MOVED') {
|
||||
kept.push(name)
|
||||
continue
|
||||
}
|
||||
// Once renamed, a fresh install at the original path is isolated from
|
||||
// deletion of the tombstone, so the sibling claim can be released.
|
||||
const release = await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host)
|
||||
gcClaimReleaseNeeded = release === 'unknown'
|
||||
await execHostCommand(conn, host, removeRemoteTreeCommand(host, tombstone))
|
||||
} catch (err) {
|
||||
if (isUnconfirmedSshCommandTermination(err)) {
|
||||
preserveGcClaim = true
|
||||
}
|
||||
throw err
|
||||
} finally {
|
||||
if (!preserveGcClaim && gcClaimReleaseNeeded) {
|
||||
await releaseRelayGcClaimWithRetry(conn, dir, gcClaimToken, host)
|
||||
}
|
||||
}
|
||||
removed.push(name)
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
|
|
@ -346,23 +312,28 @@ async function isCandidateSafeToRemove(
|
|||
): Promise<boolean> {
|
||||
const isLegacy = LEGACY_RELAY_DIR_REGEX.test(name)
|
||||
|
||||
const lockDir = joinRemotePath(host, dir, INSTALL_LOCK_NAME)
|
||||
const lockProbe = await execHostCommand(
|
||||
conn,
|
||||
host,
|
||||
probeDirectoryExistsCommand(host, lockDir)
|
||||
).catch(() => 'OPEN')
|
||||
const locked = lockProbe.trim() === 'LOCKED'
|
||||
const lockDir = joinRemotePath(host, dir, RELAY_INSTALL_LOCK_NAME)
|
||||
let lockProbe: string
|
||||
try {
|
||||
lockProbe = await execHostCommand(conn, host, probeInstallLockExistsCommand(host, lockDir))
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
const lockState = lockProbe.trim()
|
||||
if (lockState !== 'OPEN' && lockState !== 'LOCKED') {
|
||||
return false
|
||||
}
|
||||
const locked = lockState === 'LOCKED'
|
||||
|
||||
if (locked) {
|
||||
// Why: a locked dir is normally unsafe to remove — but a STALE lock
|
||||
// (mtime older than INSTALL_LOCK_STALE_MS) means the previous installer
|
||||
// (remote age older than INSTALL_LOCK_STALE_MS) means the previous installer
|
||||
// crashed and is never coming back. If the dir also has the
|
||||
// .install-complete sentinel (touch succeeded but the rm-lock at the
|
||||
// end of finalizeInstall failed), removing the dir is safe — no
|
||||
// installer is racing us, and the daemon (if any) keeps running off
|
||||
// its already-loaded code regardless of disk state.
|
||||
if (!(await isLockStale(conn, lockDir, host))) {
|
||||
if (!(await isRelayInstallLockStale(conn, lockDir, host))) {
|
||||
return false
|
||||
}
|
||||
process.stderr.write?.(`[ssh-relay] GC: lock at ${lockDir} is stale; treating as recoverable\n`)
|
||||
|
|
@ -419,8 +390,10 @@ async function hasLiveRelaySocket(
|
|||
host,
|
||||
relayLivenessProbeCommand(host, dir, windowsOptions)
|
||||
)
|
||||
return out.includes('ALIVE')
|
||||
const state = out.trim()
|
||||
return state !== 'DEAD' && state !== 'WAITING'
|
||||
} catch {
|
||||
return false
|
||||
// Why: an inconclusive liveness probe must never authorize deletion.
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,24 +1,109 @@
|
|||
import { execFileSync, spawn, spawnSync } from 'node:child_process'
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
rmSync,
|
||||
statSync,
|
||||
utimesSync
|
||||
} from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
lockAgeSecondsCommand,
|
||||
tryCreateInstallLockCommand,
|
||||
tryStealInstallLockCommand
|
||||
} from './ssh-relay-install-lock-commands'
|
||||
import {
|
||||
commandInRemoteDirectory,
|
||||
commandWithNodePath,
|
||||
listRelayBaseDirsCommand,
|
||||
makeRemoteDirectoryCommand,
|
||||
moveRemoteTreeCommand,
|
||||
probeDirectoryExistsCommand,
|
||||
probeRelayInstalledCommand,
|
||||
readRemoteHomeCommand,
|
||||
relayLivenessProbeCommand,
|
||||
tryCreateInstallLockCommand
|
||||
relayLivenessProbeCommand
|
||||
} from './ssh-remote-commands'
|
||||
import { getRemoteHostPlatform } from './ssh-remote-platform'
|
||||
|
||||
const posix = getRemoteHostPlatform('linux-x64')
|
||||
const windows = getRemoteHostPlatform('win32-x64')
|
||||
const powerShellExecutable = (
|
||||
process.platform === 'win32' ? ['pwsh.exe', 'powershell.exe'] : ['pwsh']
|
||||
).find((candidate) => {
|
||||
const result = spawnSync(candidate, ['-NoProfile', '-NonInteractive', '-Command', 'exit 0'], {
|
||||
stdio: 'ignore'
|
||||
})
|
||||
return result.status === 0
|
||||
})
|
||||
const powerShell51Executable =
|
||||
process.platform === 'win32' &&
|
||||
spawnSync('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', 'exit 0'], {
|
||||
stdio: 'ignore'
|
||||
}).status === 0
|
||||
? 'powershell.exe'
|
||||
: undefined
|
||||
|
||||
function decodePowerShellCommand(command: string): string {
|
||||
const match = command.match(/-EncodedCommand\s+([A-Za-z0-9+/=]+)/)
|
||||
return match ? Buffer.from(match[1], 'base64').toString('utf16le') : ''
|
||||
}
|
||||
|
||||
function runShellCommand(command: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('/bin/sh', ['-c', command], {
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk
|
||||
})
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk
|
||||
})
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve(stdout)
|
||||
return
|
||||
}
|
||||
reject(new Error(`shell exited ${code}: ${stderr}`))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function runPowerShellCommand(executable: string, script: string): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(executable, ['-NoProfile', '-NonInteractive', '-Command', script], {
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk) => {
|
||||
stdout += chunk
|
||||
})
|
||||
child.stderr.on('data', (chunk) => {
|
||||
stderr += chunk
|
||||
})
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve(stdout)
|
||||
return
|
||||
}
|
||||
reject(new Error(`PowerShell exited ${code}: ${stderr}`))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
describe('ssh remote command builders', () => {
|
||||
it('keeps POSIX deploy commands POSIX-native', () => {
|
||||
expect(readRemoteHomeCommand(posix)).toBe('echo $HOME')
|
||||
|
|
@ -34,7 +119,7 @@ describe('ssh remote command builders', () => {
|
|||
expect(probeRelayInstalledCommand(windows, 'C:/Users/me/relay')).toContain('-EncodedCommand')
|
||||
})
|
||||
|
||||
it('uses -Path for Windows New-Item commands', () => {
|
||||
it('uses a legacy-visible Windows lock directory with an exclusive owner file', () => {
|
||||
const mkdirScript = decodePowerShellCommand(
|
||||
makeRemoteDirectoryCommand(windows, 'C:/Users/me/.orca-remote')
|
||||
)
|
||||
|
|
@ -43,9 +128,31 @@ describe('ssh remote command builders', () => {
|
|||
)
|
||||
|
||||
expect(mkdirScript).toContain('New-Item -ItemType Directory -Force -Path')
|
||||
expect(lockScript).toContain('New-Item -ItemType Directory -Path')
|
||||
expect(lockScript).toContain('[System.IO.FileMode]::CreateNew')
|
||||
expect(lockScript).toContain('[System.IO.FileShare]::None')
|
||||
expect(lockScript).toContain('New-Item -ItemType Directory -Path $lock')
|
||||
expect(lockScript).toContain("Join-Path $lock '.owner'")
|
||||
expect(mkdirScript).not.toContain('New-Item -ItemType Directory -Force -LiteralPath')
|
||||
expect(lockScript).not.toContain('New-Item -ItemType Directory -LiteralPath')
|
||||
})
|
||||
|
||||
it('moves GC candidates to a sibling tombstone with host-native commands', () => {
|
||||
expect(moveRemoteTreeCommand(posix, '/relay/old', '/relay/old.gc-tombstone')).toContain(
|
||||
"mv '/relay/old' '/relay/old.gc-tombstone'"
|
||||
)
|
||||
const windowsScript = decodePowerShellCommand(
|
||||
moveRemoteTreeCommand(windows, 'C:/Users/me/relay/old', 'C:/Users/me/relay/old.gc-tombstone')
|
||||
)
|
||||
expect(windowsScript).toContain('Move-Item -LiteralPath')
|
||||
expect(windowsScript).toContain("-Destination 'C:/Users/me/relay/old.gc-tombstone'")
|
||||
expect(windowsScript).toContain("'MOVED'")
|
||||
})
|
||||
|
||||
it('emits an explicit POSIX liveness result so GC can fail closed', () => {
|
||||
const command = relayLivenessProbeCommand(posix, '/home/u/.orca-remote/relay-0.1.0')
|
||||
|
||||
expect(command).toContain('state=DEAD')
|
||||
expect(command).toContain('[ -S "$f" ] && state=ALIVE')
|
||||
expect(command).toContain('echo "$state"')
|
||||
})
|
||||
|
||||
it('uses named pipe try-connect liveness for Windows GC', () => {
|
||||
|
|
@ -96,11 +203,227 @@ describe('ssh remote command builders', () => {
|
|||
tryCreateInstallLockCommand(windows, 'C:/Users/me/.orca-remote/relay/.install-lock')
|
||||
)
|
||||
|
||||
expect(script).toContain('$ErrorActionPreference = "Stop"; try {')
|
||||
expect(script).toContain('$stream = $null; try {')
|
||||
expect(script).toContain("} catch { 'BUSY' }")
|
||||
expect(script).not.toContain('}; catch')
|
||||
})
|
||||
|
||||
it('computes install-lock age on the remote host clock', () => {
|
||||
const posixCommand = lockAgeSecondsCommand(posix, '/home/me/.orca-remote/relay/.install-lock')
|
||||
const windowsScript = decodePowerShellCommand(
|
||||
lockAgeSecondsCommand(windows, 'C:/Users/me/.orca-remote/relay/.install-lock')
|
||||
)
|
||||
|
||||
expect(posixCommand).toContain('date +%s')
|
||||
expect(posixCommand).toContain('echo "$age"')
|
||||
expect(windowsScript).toContain('[DateTimeOffset]::UtcNow.ToUnixTimeSeconds()')
|
||||
expect(windowsScript).toContain('Write-Output ($now - $mtime)')
|
||||
})
|
||||
|
||||
it('serializes stale recovery with unbounded numbered sibling claims', () => {
|
||||
const posixCommand = tryStealInstallLockCommand(
|
||||
posix,
|
||||
'/home/me/.orca-remote/relay/.install-lock',
|
||||
20 * 60
|
||||
)
|
||||
const windowsScript = decodePowerShellCommand(
|
||||
tryStealInstallLockCommand(windows, 'C:/Users/me/.orca-remote/relay/.install-lock', 20 * 60)
|
||||
)
|
||||
|
||||
expect(posixCommand).toContain('.install-lock')
|
||||
expect(posixCommand).toContain('.install-lock.steal')
|
||||
expect(posixCommand).toContain('steal_generation')
|
||||
expect(posixCommand).toContain('mtime=${lock_key%%:*}')
|
||||
expect(posixCommand).toContain('-gt 1200')
|
||||
expect(posixCommand).toContain('current_mtime')
|
||||
expect(posixCommand).toContain('steal_generation + 1')
|
||||
expect(posixCommand).not.toContain('.next.')
|
||||
expect(posixCommand).toContain('trap')
|
||||
expect(windowsScript).toContain('$lock.steal')
|
||||
expect(windowsScript).toContain('$stealGeneration++')
|
||||
expect(windowsScript).toContain('-gt 1200')
|
||||
expect(windowsScript).toContain('$currentIdentity -eq $lockIdentity')
|
||||
expect(windowsScript).toContain('[System.IO.FileMode]::CreateNew')
|
||||
expect(windowsScript).not.toContain('.next.')
|
||||
expect(windowsScript).toContain('finally')
|
||||
})
|
||||
|
||||
it.runIf(powerShellExecutable)(
|
||||
'emits a parseable Windows stale-lock recovery command',
|
||||
() => {
|
||||
const script = decodePowerShellCommand(
|
||||
tryStealInstallLockCommand(
|
||||
windows,
|
||||
'C:/Users/orca-missing/.orca-remote/relay/.install-lock',
|
||||
20 * 60
|
||||
)
|
||||
)
|
||||
const result = spawnSync(
|
||||
powerShellExecutable!,
|
||||
['-NoProfile', '-NonInteractive', '-Command', script],
|
||||
{ encoding: 'utf8' }
|
||||
)
|
||||
|
||||
expect(result.status, result.stderr).toBe(0)
|
||||
expect(result.stdout.trim()).toBe('BUSY')
|
||||
},
|
||||
15_000
|
||||
)
|
||||
|
||||
it.runIf(powerShell51Executable)(
|
||||
'lets only one Windows PowerShell 5.1 caller acquire an install lock',
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-windows-race-'))
|
||||
try {
|
||||
const lockPath = join(root, '.install-lock')
|
||||
const script = decodePowerShellCommand(tryCreateInstallLockCommand(windows, lockPath))
|
||||
const outputs = await Promise.all(
|
||||
Array.from({ length: 16 }, () => runPowerShellCommand(powerShell51Executable!, script))
|
||||
)
|
||||
|
||||
expect(outputs.filter((output) => output.trim().endsWith('OK'))).toHaveLength(1)
|
||||
expect(statSync(lockPath).isDirectory()).toBe(true)
|
||||
expect(statSync(join(lockPath, '.owner')).isFile()).toBe(true)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
},
|
||||
30_000
|
||||
)
|
||||
|
||||
it.runIf(powerShell51Executable)(
|
||||
'recovers a stale Windows lock past abandoned steal generations',
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-windows-stale-'))
|
||||
try {
|
||||
const lockPath = join(root, '.install-lock')
|
||||
mkdirSync(lockPath)
|
||||
const staleDate = new Date(Date.now() - 60 * 60_000)
|
||||
utimesSync(lockPath, staleDate, staleDate)
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const orphan = `${lockPath}.steal.${i}`
|
||||
mkdirSync(orphan)
|
||||
utimesSync(orphan, staleDate, staleDate)
|
||||
}
|
||||
const script = decodePowerShellCommand(
|
||||
tryStealInstallLockCommand(windows, lockPath, 20 * 60)
|
||||
)
|
||||
|
||||
const output = await runPowerShellCommand(powerShell51Executable!, script)
|
||||
|
||||
expect(output.trim()).toBe('OK')
|
||||
expect(statSync(lockPath).isDirectory()).toBe(true)
|
||||
expect(statSync(join(lockPath, '.owner')).isFile()).toBe(true)
|
||||
expect(readdirSync(root).filter((name) => name.includes('.steal.'))).toHaveLength(0)
|
||||
expect(readdirSync(root).filter((name) => name.includes('.tombstone.'))).toHaveLength(0)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
},
|
||||
15_000
|
||||
)
|
||||
|
||||
it.runIf(powerShell51Executable)(
|
||||
'lets only one Windows PowerShell 5.1 caller replace a stale lock',
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-windows-steal-race-'))
|
||||
try {
|
||||
const lockPath = join(root, '.install-lock')
|
||||
mkdirSync(lockPath)
|
||||
const staleDate = new Date(Date.now() - 60 * 60_000)
|
||||
utimesSync(lockPath, staleDate, staleDate)
|
||||
const script = decodePowerShellCommand(
|
||||
tryStealInstallLockCommand(windows, lockPath, 20 * 60)
|
||||
)
|
||||
|
||||
const outputs = await Promise.all(
|
||||
Array.from({ length: 16 }, () => runPowerShellCommand(powerShell51Executable!, script))
|
||||
)
|
||||
|
||||
expect(outputs.filter((output) => output.trim().endsWith('OK'))).toHaveLength(1)
|
||||
expect(statSync(lockPath).isDirectory()).toBe(true)
|
||||
expect(statSync(join(lockPath, '.owner')).isFile()).toBe(true)
|
||||
expect(readdirSync(root).filter((name) => name.includes('.tombstone.'))).toHaveLength(0)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
},
|
||||
30_000
|
||||
)
|
||||
|
||||
it.runIf(powerShellExecutable)(
|
||||
'keeps a new Windows lock visible to the previous directory-only GC probe',
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-windows-compat-'))
|
||||
try {
|
||||
const lockPath = join(root, '.install-lock')
|
||||
const acquire = decodePowerShellCommand(tryCreateInstallLockCommand(windows, lockPath))
|
||||
const legacyProbe = decodePowerShellCommand(probeDirectoryExistsCommand(windows, lockPath))
|
||||
|
||||
await expect(runPowerShellCommand(powerShellExecutable!, acquire)).resolves.toMatch(/OK/)
|
||||
await expect(runPowerShellCommand(powerShellExecutable!, legacyProbe)).resolves.toMatch(
|
||||
/LOCKED/
|
||||
)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
},
|
||||
15_000
|
||||
)
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'recovers and cleans more than eight orphaned numbered steal claims',
|
||||
() => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-'))
|
||||
try {
|
||||
const lockDir = join(root, '.install-lock')
|
||||
mkdirSync(lockDir)
|
||||
const staleDate = new Date(Date.now() - 60 * 60_000)
|
||||
utimesSync(lockDir, staleDate, staleDate)
|
||||
const lockMtimeSeconds = Math.floor(statSync(lockDir).mtimeMs / 1000)
|
||||
for (let i = 0; i < 12; i++) {
|
||||
const orphan = `${lockDir}.steal.${i}`
|
||||
mkdirSync(orphan)
|
||||
utimesSync(orphan, staleDate, staleDate)
|
||||
}
|
||||
|
||||
const command = tryStealInstallLockCommand(posix, lockDir, 20 * 60)
|
||||
const output = execFileSync('/bin/sh', ['-c', command], { encoding: 'utf8' })
|
||||
|
||||
expect(output.trim()).toBe('OK')
|
||||
expect(existsSync(lockDir)).toBe(true)
|
||||
expect(readdirSync(root).filter((name) => name.includes('.steal.'))).toHaveLength(0)
|
||||
expect(Math.floor(statSync(lockDir).mtimeMs / 1000)).toBeGreaterThan(lockMtimeSeconds)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it.runIf(process.platform !== 'win32')(
|
||||
'lets only one POSIX caller move and recreate a stale install lock',
|
||||
async () => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'orca-install-lock-race-'))
|
||||
try {
|
||||
const lockDir = join(root, '.install-lock')
|
||||
mkdirSync(lockDir)
|
||||
const staleDate = new Date(Date.now() - 60 * 60_000)
|
||||
utimesSync(lockDir, staleDate, staleDate)
|
||||
const command = tryStealInstallLockCommand(posix, lockDir, 20 * 60)
|
||||
const outputs = await Promise.all(
|
||||
Array.from({ length: 64 }, () => runShellCommand(command))
|
||||
)
|
||||
const okCount = outputs.filter((output) => output.trim().endsWith('OK')).length
|
||||
|
||||
expect(okCount).toBe(1)
|
||||
expect(existsSync(lockDir)).toBe(true)
|
||||
expect(readdirSync(root).some((name) => name.includes('.tombstone'))).toBe(false)
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('makes Windows remote directory changes fail before running scoped commands', () => {
|
||||
const scopedCommand = decodePowerShellCommand(
|
||||
commandInRemoteDirectory(windows, 'C:/Users/me/.orca-remote/relay-0.1.0', "'READY'")
|
||||
|
|
|
|||
|
|
@ -45,6 +45,24 @@ export function removeRemoteTreeCommand(host: RemoteHostPlatform, remotePath: st
|
|||
)
|
||||
}
|
||||
|
||||
export function moveRemoteTreeCommand(
|
||||
host: RemoteHostPlatform,
|
||||
sourcePath: string,
|
||||
destinationPath: string
|
||||
): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `mv ${shellEscape(sourcePath)} ${shellEscape(destinationPath)} 2>&1 && echo MOVED || echo BUSY`
|
||||
}
|
||||
return powerShellCommand(
|
||||
[
|
||||
'try {',
|
||||
`Move-Item -LiteralPath ${powerShellLiteral(sourcePath)} -Destination ${powerShellLiteral(destinationPath)} -ErrorAction Stop`,
|
||||
"'MOVED'",
|
||||
`} catch { 'BUSY' }`
|
||||
].join('; ')
|
||||
)
|
||||
}
|
||||
|
||||
export function writeRemoteEmptyFileCommand(host: RemoteHostPlatform, remotePath: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `touch ${shellEscape(remotePath)}`
|
||||
|
|
@ -81,36 +99,6 @@ export function probeRelayInstalledCommand(
|
|||
)
|
||||
}
|
||||
|
||||
export function acquireInstallLockParentCommand(
|
||||
host: RemoteHostPlatform,
|
||||
remoteRelayDir: string
|
||||
): string {
|
||||
return makeRemoteDirectoryCommand(host, remoteRelayDir)
|
||||
}
|
||||
|
||||
export function tryCreateInstallLockCommand(host: RemoteHostPlatform, lockDir: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `mkdir ${shellEscape(lockDir)} 2>&1 && echo OK || echo BUSY`
|
||||
}
|
||||
// New-Item has no -LiteralPath parameter; using it breaks stock Windows PowerShell.
|
||||
return powerShellCommand(
|
||||
`$ErrorActionPreference = "Stop"; try { $null = New-Item -ItemType Directory -Path ${powerShellLiteral(lockDir)}; 'OK' } catch { 'BUSY' }`
|
||||
)
|
||||
}
|
||||
|
||||
export function lockMtimeEpochCommand(host: RemoteHostPlatform, lockDir: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `stat -c %Y ${shellEscape(lockDir)} 2>/dev/null || stat -f %m ${shellEscape(lockDir)} 2>/dev/null || echo`
|
||||
}
|
||||
return powerShellCommand(
|
||||
[
|
||||
`$item = Get-Item -LiteralPath ${powerShellLiteral(lockDir)} -ErrorAction Stop`,
|
||||
'$dto = [DateTimeOffset]$item.LastWriteTimeUtc',
|
||||
'Write-Output $dto.ToUnixTimeSeconds()'
|
||||
].join('; ')
|
||||
)
|
||||
}
|
||||
|
||||
export function listRelayBaseDirsCommand(host: RemoteHostPlatform, baseDir: string): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return `ls -1 ${shellEscape(baseDir)} 2>/dev/null || true`
|
||||
|
|
@ -155,9 +143,9 @@ export function relayLivenessProbeCommand(
|
|||
): string {
|
||||
if (!isWindowsRemoteHost(host)) {
|
||||
return (
|
||||
`for f in ${shellEscape(dir)}/relay-*.sock ${shellEscape(dir)}/relay.sock; do ` +
|
||||
`[ -S "$f" ] && echo ALIVE && break; ` +
|
||||
'done; true'
|
||||
`state=DEAD; for f in ${shellEscape(dir)}/relay-*.sock ${shellEscape(dir)}/relay.sock; do ` +
|
||||
`[ -S "$f" ] && state=ALIVE && break; ` +
|
||||
'done; echo "$state"'
|
||||
)
|
||||
}
|
||||
if (!windowsOptions) {
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ true
|
|||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
throwIfAborted(options)
|
||||
// Fall through to login shell.
|
||||
}
|
||||
return null
|
||||
|
|
@ -167,6 +168,7 @@ async function tryResolveViaLoginShell(
|
|||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
throwIfAborted(options)
|
||||
// Fall through.
|
||||
}
|
||||
return null
|
||||
|
|
@ -191,6 +193,7 @@ async function nodeMeetsVersionRequirement(
|
|||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
throwIfAborted(options)
|
||||
// Binary missing or fails to run — not usable.
|
||||
return false
|
||||
}
|
||||
|
|
@ -240,6 +243,7 @@ async function resolveRemoteWindowsNodePath(
|
|||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
throwIfAborted(options)
|
||||
// Fall through to the shared error below.
|
||||
}
|
||||
|
||||
|
|
@ -262,6 +266,7 @@ async function windowsNodeMeetsVersionRequirement(
|
|||
if (options?.rethrowSessionLimitErrors && isSshSessionLimitError(err)) {
|
||||
throw err
|
||||
}
|
||||
throwIfAborted(options)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,30 +11,38 @@ import { powerShellCommand } from './ssh-remote-powershell'
|
|||
const PLATFORM_PROBE_MARKER = '__ORCA_REMOTE_PLATFORM__'
|
||||
|
||||
export async function detectRemoteHostPlatform(
|
||||
conn: SshConnection
|
||||
conn: SshConnection,
|
||||
options?: { signal?: AbortSignal }
|
||||
): Promise<RemoteHostPlatform | null> {
|
||||
const unamePlatform = await detectUnamePlatform(conn)
|
||||
const unamePlatform = await detectUnamePlatform(conn, options?.signal)
|
||||
if (unamePlatform) {
|
||||
return getRemoteHostPlatform(unamePlatform)
|
||||
}
|
||||
const windowsPlatform = await detectWindowsPlatform(conn)
|
||||
const windowsPlatform = await detectWindowsPlatform(conn, options?.signal)
|
||||
return windowsPlatform ? getRemoteHostPlatform(windowsPlatform) : null
|
||||
}
|
||||
|
||||
async function detectUnamePlatform(conn: SshConnection): Promise<RelayPlatform | null> {
|
||||
async function detectUnamePlatform(
|
||||
conn: SshConnection,
|
||||
signal?: AbortSignal
|
||||
): Promise<RelayPlatform | null> {
|
||||
try {
|
||||
const output = await execCommand(
|
||||
conn,
|
||||
// Why: Remote startup output may omit its trailing newline and must not absorb the marker.
|
||||
`printf '\\n%s ' '${PLATFORM_PROBE_MARKER}'; uname -sm`
|
||||
)
|
||||
// Why: Remote startup output may omit its trailing newline and must not absorb the marker.
|
||||
const command = `printf '\\n%s ' '${PLATFORM_PROBE_MARKER}'; uname -sm`
|
||||
const output = signal
|
||||
? await execCommand(conn, command, { signal })
|
||||
: await execCommand(conn, command)
|
||||
return parseRemotePlatformOutput(output)
|
||||
} catch {
|
||||
signal?.throwIfAborted()
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
async function detectWindowsPlatform(conn: SshConnection): Promise<RelayPlatform | null> {
|
||||
async function detectWindowsPlatform(
|
||||
conn: SshConnection,
|
||||
signal?: AbortSignal
|
||||
): Promise<RelayPlatform | null> {
|
||||
try {
|
||||
const script = [
|
||||
'$arch = $env:PROCESSOR_ARCHITECTURE',
|
||||
|
|
@ -43,9 +51,13 @@ async function detectWindowsPlatform(conn: SshConnection): Promise<RelayPlatform
|
|||
// Why: Remote startup output may omit its trailing newline and must not absorb the marker.
|
||||
`Write-Output ("\`n${PLATFORM_PROBE_MARKER} Windows " + $arch)`
|
||||
].join('; ')
|
||||
const output = await execCommand(conn, powerShellCommand(script), { wrapCommand: false })
|
||||
const output = await execCommand(conn, powerShellCommand(script), {
|
||||
wrapCommand: false,
|
||||
...(signal ? { signal } : {})
|
||||
})
|
||||
return parseRemotePlatformOutput(output)
|
||||
} catch {
|
||||
signal?.throwIfAborted()
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -83,6 +83,16 @@ function expectNoOrcaControlMasterArgs(args: string[]): void {
|
|||
expect(args).not.toContain('ControlPersist=300')
|
||||
}
|
||||
|
||||
function expectOrcaControlMasterArgs(args: string[]): void {
|
||||
if (process.platform === 'win32') {
|
||||
expectNoOrcaControlMasterArgs(args)
|
||||
return
|
||||
}
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args.some((arg) => arg.startsWith('ControlPath='))).toBe(true)
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
}
|
||||
|
||||
type EventedProcess = EventEmitter & {
|
||||
stdin: EventEmitter & {
|
||||
write: ReturnType<typeof vi.fn>
|
||||
|
|
@ -360,9 +370,7 @@ describe('spawnSystemSsh', () => {
|
|||
})
|
||||
})
|
||||
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args.some((arg) => arg.startsWith('ControlPath='))).toBe(true)
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
expectOrcaControlMasterArgs(args)
|
||||
expect(args).not.toContain('-S')
|
||||
})
|
||||
|
||||
|
|
@ -374,9 +382,7 @@ describe('spawnSystemSsh', () => {
|
|||
})
|
||||
})
|
||||
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args.some((arg) => arg.startsWith('ControlPath='))).toBe(true)
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
expectOrcaControlMasterArgs(args)
|
||||
expect(args).not.toContain('-S')
|
||||
})
|
||||
|
||||
|
|
@ -387,9 +393,7 @@ describe('spawnSystemSsh', () => {
|
|||
})
|
||||
})
|
||||
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args.some((arg) => arg.startsWith('ControlPath='))).toBe(true)
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
expectOrcaControlMasterArgs(args)
|
||||
expect(args).not.toContain('-S')
|
||||
})
|
||||
|
||||
|
|
@ -414,9 +418,7 @@ describe('spawnSystemSsh', () => {
|
|||
resolvedConfig: createResolvedConfig()
|
||||
})
|
||||
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args.some((arg) => arg.startsWith('ControlPath='))).toBe(true)
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
expectOrcaControlMasterArgs(args)
|
||||
expect(args).not.toContain('-S')
|
||||
})
|
||||
|
||||
|
|
@ -432,10 +434,11 @@ describe('spawnSystemSsh', () => {
|
|||
it('adds keepalive options to Orca-owned ControlMaster connections', () => {
|
||||
const args = buildSshArgs(createTarget(), { resolvedConfig: createResolvedConfig() })
|
||||
|
||||
expect(args).toContain('ControlMaster=auto')
|
||||
expect(args).toContain('ControlPersist=300')
|
||||
expect(args).toContain('ServerAliveInterval=15')
|
||||
expect(args).toContain('ServerAliveCountMax=3')
|
||||
expectOrcaControlMasterArgs(args)
|
||||
if (process.platform !== 'win32') {
|
||||
expect(args).toContain('ServerAliveInterval=15')
|
||||
expect(args).toContain('ServerAliveCountMax=3')
|
||||
}
|
||||
})
|
||||
|
||||
it('spawns a remote command through the system ssh target', () => {
|
||||
|
|
@ -507,6 +510,15 @@ describe('spawnSystemSsh', () => {
|
|||
expect(mockProc.stdin.end).toHaveBeenCalledWith('contents')
|
||||
})
|
||||
|
||||
it('marks a system command channel when local teardown is requested', () => {
|
||||
const channel = spawnSystemSshCommand(createTarget(), 'npm install')
|
||||
|
||||
channel.close()
|
||||
|
||||
expect(channel._closeRequested).toBe(true)
|
||||
expect(mockProc.kill).toHaveBeenCalledWith('SIGTERM')
|
||||
})
|
||||
|
||||
it('removes wrapped process listeners after command close', () => {
|
||||
const proc = createEventedProcess()
|
||||
spawnMock.mockReturnValue(proc)
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ export type SystemSshProcess = {
|
|||
|
||||
export type SystemSshCommandChannel = ClientChannel & {
|
||||
_process?: ChildProcess
|
||||
_closeRequested?: boolean
|
||||
}
|
||||
|
||||
type SystemSshCommandOptions = SshExecOptions & SystemSshBuildArgsOptions
|
||||
|
|
@ -103,12 +104,14 @@ function wrapCommandProcess(proc: ChildProcess): SystemSshCommandChannel {
|
|||
stdin: NodeJS.WritableStream
|
||||
stderr: NodeJS.ReadableStream
|
||||
_process?: ChildProcess
|
||||
_closeRequested?: boolean
|
||||
close: () => void
|
||||
}
|
||||
mutableChannel.stdin = proc.stdin!
|
||||
mutableChannel.stderr = proc.stderr!
|
||||
mutableChannel._process = proc
|
||||
mutableChannel.close = () => {
|
||||
mutableChannel._closeRequested = true
|
||||
try {
|
||||
proc.kill('SIGTERM')
|
||||
} catch {
|
||||
|
|
|
|||
|
|
@ -100,7 +100,13 @@ export async function awaitWithSystemSshAbort<T>(
|
|||
// children to emit close after we've already signaled them.
|
||||
abortChildren()
|
||||
suppressLateOperationError = true
|
||||
abortReject?.(createAbortError())
|
||||
abortReject?.(
|
||||
Object.assign(createAbortError(), {
|
||||
// The child was signaled, but this fast abort path intentionally does
|
||||
// not wait for process exit; callers holding remote locks must retain them.
|
||||
sshChannelCloseConfirmed: false
|
||||
})
|
||||
)
|
||||
}
|
||||
signal.addEventListener('abort', abort, { once: true })
|
||||
if (signal.aborted) {
|
||||
|
|
|
|||
|
|
@ -181,6 +181,29 @@ describe('PtyHandler', () => {
|
|||
expect(handler.activePtyCount).toBe(1)
|
||||
})
|
||||
|
||||
it('normalizes a missing native binding as degraded node-pty availability', async () => {
|
||||
mockPtySpawn.mockImplementationOnce(() => {
|
||||
throw new Error(
|
||||
'Failed to load native module: conpty.node, checked: build/Release, prebuilds/win32-x64'
|
||||
)
|
||||
})
|
||||
|
||||
await expect(dispatcher.callRequest('pty.spawn', {})).rejects.toThrow(
|
||||
'node-pty is not available on this remote host'
|
||||
)
|
||||
expect(handler.activePtyCount).toBe(0)
|
||||
})
|
||||
|
||||
it('preserves unrelated node-pty spawn failures', async () => {
|
||||
mockPtySpawn.mockImplementationOnce(() => {
|
||||
throw new Error('File not found: missing-shell.exe')
|
||||
})
|
||||
|
||||
await expect(dispatcher.callRequest('pty.spawn', {})).rejects.toThrow(
|
||||
'File not found: missing-shell.exe'
|
||||
)
|
||||
})
|
||||
|
||||
it('atomically caps concurrent PTY spawn admission', async () => {
|
||||
const results = await Promise.allSettled(
|
||||
Array.from({ length: MAX_RELAY_PTY_SESSIONS + 1 }, () =>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
/* oxlint-disable max-lines */
|
||||
import type { IPty } from 'node-pty'
|
||||
import type * as NodePty from 'node-pty'
|
||||
import { existsSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { resolveWindowsGitBashShellPath } from '../main/git-bash'
|
||||
import { WINDOWS_GIT_BASH_SHELL } from '../shared/windows-terminal-shell'
|
||||
import type { RelayDispatcher, RequestContext } from './dispatcher'
|
||||
|
|
@ -38,21 +40,11 @@ import {
|
|||
import { isTuiAgent } from '../shared/tui-agent-config'
|
||||
import { forceKillPosixPtyProcessGroups } from '../main/pty/posix-pty-process-groups'
|
||||
|
||||
// Why: node-pty is a native addon that may not be installed on the remote.
|
||||
// Dynamic import keeps the require() lazy so loadPty() returns null gracefully
|
||||
// when the native module is unavailable. The static type import lets vitest
|
||||
// intercept it in tests.
|
||||
let ptyModule: typeof NodePty | null = null
|
||||
async function loadPty(): Promise<typeof NodePty | null> {
|
||||
if (ptyModule) {
|
||||
return ptyModule
|
||||
}
|
||||
try {
|
||||
ptyModule = await import('node-pty')
|
||||
return ptyModule
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
function isMissingNodePtyNativeBinding(error: unknown): boolean {
|
||||
return (
|
||||
error instanceof Error &&
|
||||
/Failed to load native module: (?:conpty|pty)\.node(?:,|$)/.test(error.message)
|
||||
)
|
||||
}
|
||||
|
||||
type ManagedPty = {
|
||||
|
|
@ -296,6 +288,9 @@ export class PtyHandler {
|
|||
private pendingCreationDrainResolvers = new Set<() => void>()
|
||||
private worktreeRemovalCoordinator: RelayPtyWorktreeRemovalCoordinator | null = null
|
||||
private disposePromise: Promise<void> | null = null
|
||||
private ptyModule: typeof NodePty | null = null
|
||||
private ptyModuleLoadPromise: Promise<typeof NodePty | null> | null = null
|
||||
private reloadPtyModuleFromDisk = false
|
||||
// Why: external observers need to drop per-pane state when a PTY exits.
|
||||
// Today the relay composes multiple consumers (hook-server cache eviction
|
||||
// and plugin-overlay dir cleanup) into a single callback at the call site
|
||||
|
|
@ -316,6 +311,55 @@ export class PtyHandler {
|
|||
this.registerHandlers()
|
||||
}
|
||||
|
||||
private async loadPty(): Promise<typeof NodePty | null> {
|
||||
if (this.ptyModule) {
|
||||
return this.ptyModule
|
||||
}
|
||||
if (this.ptyModuleLoadPromise) {
|
||||
return this.ptyModuleLoadPromise
|
||||
}
|
||||
this.ptyModuleLoadPromise = this.loadPtyUncached()
|
||||
try {
|
||||
return await this.ptyModuleLoadPromise
|
||||
} finally {
|
||||
this.ptyModuleLoadPromise = null
|
||||
}
|
||||
}
|
||||
|
||||
private async loadPtyUncached(): Promise<typeof NodePty | null> {
|
||||
if (!this.reloadPtyModuleFromDisk) {
|
||||
try {
|
||||
this.ptyModule = await import('node-pty')
|
||||
return this.ptyModule
|
||||
} catch {
|
||||
this.reloadPtyModuleFromDisk = true
|
||||
}
|
||||
}
|
||||
// Why: the relay is launched from its install dir today, but module
|
||||
// resolution must remain tied to the deployed bundle rather than cwd.
|
||||
const moduleEntry = join(__dirname, 'node_modules', 'node-pty', 'lib', 'index.js')
|
||||
if (!existsSync(moduleEntry)) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
this.ptyModule = require(moduleEntry) as typeof NodePty
|
||||
return this.ptyModule
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private invalidatePtyModuleAfterBindingFailure(): void {
|
||||
this.ptyModule = null
|
||||
this.reloadPtyModuleFromDisk = true
|
||||
const moduleRoot = join(__dirname, 'node_modules', 'node-pty')
|
||||
for (const cachedPath of Object.keys(require.cache)) {
|
||||
if (isPathInsideOrEqual(moduleRoot, cachedPath)) {
|
||||
delete require.cache[cachedPath]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setGraceTimeMs(graceTimeMs: number): void {
|
||||
this.graceTimeMs = Math.max(0, Math.floor(graceTimeMs))
|
||||
}
|
||||
|
|
@ -741,7 +785,7 @@ export class PtyHandler {
|
|||
params: Record<string, unknown>,
|
||||
context?: RequestContext
|
||||
): Promise<{ id: string }> {
|
||||
const pty = await loadPty()
|
||||
const pty = await this.loadPty()
|
||||
if (!pty) {
|
||||
throw new Error('node-pty is not available on this remote host')
|
||||
}
|
||||
|
|
@ -814,17 +858,28 @@ export class PtyHandler {
|
|||
// includes Homebrew, nvm, and user-installed CLIs (claude, codex, gh).
|
||||
// When overlays are injected, the launch wrapper keeps those paths after
|
||||
// user startup files re-export their defaults.
|
||||
const term = pty.spawn(shell, shellLaunch.args, {
|
||||
// Why: node-pty overwrites env.TERM with `name`; keep caller-selected
|
||||
// terminal identities instead of losing them at the final spawn boundary.
|
||||
name: spawnEnv.TERM ?? 'xterm-256color',
|
||||
cols,
|
||||
rows,
|
||||
cwd,
|
||||
// Why: relay shells inherit process.env; never let an ambient Orca marker
|
||||
// enable shell-ready behavior unless this spawn explicitly requested it.
|
||||
env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env }
|
||||
})
|
||||
let term: IPty
|
||||
try {
|
||||
term = pty.spawn(shell, shellLaunch.args, {
|
||||
// Why: node-pty overwrites env.TERM with `name`; keep caller-selected
|
||||
// terminal identities instead of losing them at the final spawn boundary.
|
||||
name: spawnEnv.TERM ?? 'xterm-256color',
|
||||
cols,
|
||||
rows,
|
||||
cwd,
|
||||
// Why: relay shells inherit process.env; never let an ambient Orca marker
|
||||
// enable shell-ready behavior unless this spawn explicitly requested it.
|
||||
env: { ...spawnEnv, ORCA_SHELL_READY_MARKER: '0', ...shellLaunch.env }
|
||||
})
|
||||
} catch (error) {
|
||||
// Why: Windows node-pty loads conpty.node only on first spawn, after the
|
||||
// wrapper import succeeded. Keep that late failure on the degraded path.
|
||||
if (isMissingNodePtyNativeBinding(error)) {
|
||||
this.invalidatePtyModuleAfterBindingFailure()
|
||||
throw new Error('node-pty is not available on this remote host')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
|
||||
// Why: capture the renderer-supplied paneKey on the managed entry so the
|
||||
// exit listener can evict per-pane caches without the relay needing a
|
||||
|
|
@ -1215,7 +1270,7 @@ export class PtyHandler {
|
|||
}
|
||||
|
||||
private async reviveEntry(entry: SerializedPtyEntry): Promise<void> {
|
||||
const ptyMod = await loadPty()
|
||||
const ptyMod = await this.loadPty()
|
||||
if (!ptyMod) {
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ export type RelayProcess = {
|
|||
export function spawnRelay(
|
||||
entryPath: string,
|
||||
args: string[] = [],
|
||||
options: Pick<SpawnOptions, 'env'> = {}
|
||||
options: Pick<SpawnOptions, 'cwd' | 'env'> = {}
|
||||
): RelayProcess {
|
||||
const proc = spawn('node', [entryPath, ...args], {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
|
|
|
|||
|
|
@ -1,5 +1,13 @@
|
|||
import { afterAll, beforeAll, describe, expect, it, afterEach } from 'vitest'
|
||||
import { existsSync, mkdtempSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'
|
||||
import {
|
||||
copyFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
unlinkSync,
|
||||
writeFileSync
|
||||
} from 'node:fs'
|
||||
import { rm } from 'node:fs/promises'
|
||||
import * as path from 'node:path'
|
||||
import { tmpdir } from 'node:os'
|
||||
|
|
@ -46,9 +54,14 @@ afterAll(async () => {
|
|||
}
|
||||
})
|
||||
|
||||
function spawn(args: string[] = [], env?: NodeJS.ProcessEnv): RelayProcess {
|
||||
function spawnRelayEntry(
|
||||
entryPath: string,
|
||||
args: string[] = [],
|
||||
env?: NodeJS.ProcessEnv
|
||||
): RelayProcess {
|
||||
let relayArgs = args
|
||||
if (!args.includes('--sock-path')) {
|
||||
// Why: Windows relays require a named pipe; filesystem socket paths fail with EACCES.
|
||||
const socketDir = mkdtempSync(path.join(tmpdir(), 'relay-sock-'))
|
||||
spawnedSocketDirs.push(socketDir)
|
||||
relayArgs = [
|
||||
|
|
@ -59,7 +72,11 @@ function spawn(args: string[] = [], env?: NodeJS.ProcessEnv): RelayProcess {
|
|||
path.join(socketDir, 'agent-hooks')
|
||||
]
|
||||
}
|
||||
return spawnRelay(relayEntry, relayArgs, env ? { env } : undefined)
|
||||
return spawnRelay(entryPath, relayArgs, env ? { env } : undefined)
|
||||
}
|
||||
|
||||
function spawn(args: string[] = [], env?: NodeJS.ProcessEnv): RelayProcess {
|
||||
return spawnRelayEntry(relayEntry, args, env)
|
||||
}
|
||||
|
||||
function waitForChildExit(
|
||||
|
|
@ -75,6 +92,22 @@ function waitForChildExit(
|
|||
})
|
||||
}
|
||||
|
||||
function writeMockNodePty(root: string, source: string, withPackageEntry = false): void {
|
||||
const nodePtyDir = path.join(root, 'node_modules', 'node-pty')
|
||||
const libDir = path.join(nodePtyDir, 'lib')
|
||||
mkdirSync(libDir, { recursive: true })
|
||||
if (withPackageEntry) {
|
||||
writeFileSync(path.join(nodePtyDir, 'package.json'), '{"main":"lib/index.js"}\n')
|
||||
}
|
||||
writeFileSync(path.join(libDir, 'index.js'), source)
|
||||
}
|
||||
|
||||
const WORKING_NODE_PTY_MODULE = `module.exports = { spawn() { return {
|
||||
pid: process.pid,
|
||||
process: 'mock-shell',
|
||||
onData() {}, onExit() {}, write() {}, resize() {}, kill() {}, clear() {}
|
||||
} } }\n`
|
||||
|
||||
describe('Subprocess: Relay entry point', () => {
|
||||
let relay: RelayProcess | null = null
|
||||
let tmpDir: string
|
||||
|
|
@ -103,6 +136,50 @@ describe('Subprocess: Relay entry point', () => {
|
|||
expect(readFileSync(relayEntry, 'utf8')).not.toContain('.toReversed(')
|
||||
})
|
||||
|
||||
it('loads node-pty after an in-place dependency repair without restarting', async () => {
|
||||
tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-native-repair-'))
|
||||
const repairedRelayEntry = path.join(tmpDir, 'relay.js')
|
||||
copyFileSync(relayEntry, repairedRelayEntry)
|
||||
|
||||
relay = spawnRelayEntry(repairedRelayEntry)
|
||||
await relay.sentinelReceived
|
||||
|
||||
const failedId = relay.send('pty.spawn', { cols: 80, rows: 24 })
|
||||
const failed = await relay.waitForResponse(failedId)
|
||||
expect(failed.error?.message).toContain('node-pty is not available')
|
||||
|
||||
writeMockNodePty(tmpDir, WORKING_NODE_PTY_MODULE)
|
||||
|
||||
const repairedId = relay.send('pty.spawn', { cols: 80, rows: 24 })
|
||||
const repaired = await relay.waitForResponse(repairedId)
|
||||
expect(repaired.error).toBeUndefined()
|
||||
expect(repaired.result).toMatchObject({ id: 'pty-1' })
|
||||
}, 10_000)
|
||||
|
||||
it('reloads node-pty after a late native binding failure without restarting', async () => {
|
||||
tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-native-late-repair-'))
|
||||
const repairedRelayEntry = path.join(tmpDir, 'relay.js')
|
||||
copyFileSync(relayEntry, repairedRelayEntry)
|
||||
writeMockNodePty(
|
||||
tmpDir,
|
||||
`module.exports = { spawn() { throw new Error('Failed to load native module: conpty.node, checked: prebuilds/win32-x64') } }\n`,
|
||||
true
|
||||
)
|
||||
|
||||
relay = spawnRelayEntry(repairedRelayEntry)
|
||||
await relay.sentinelReceived
|
||||
|
||||
const failedId = relay.send('pty.spawn', { cols: 80, rows: 24 })
|
||||
const failed = await relay.waitForResponse(failedId)
|
||||
expect(failed.error?.message).toContain('node-pty is not available')
|
||||
|
||||
writeMockNodePty(tmpDir, WORKING_NODE_PTY_MODULE, true)
|
||||
const repairedId = relay.send('pty.spawn', { cols: 80, rows: 24 })
|
||||
const repaired = await relay.waitForResponse(repairedId)
|
||||
expect(repaired.error).toBeUndefined()
|
||||
expect(repaired.result).toMatchObject({ id: 'pty-2' })
|
||||
}, 10_000)
|
||||
|
||||
it('responds to fs.stat over stdin/stdout', async () => {
|
||||
tmpDir = mkdtempSync(path.join(tmpdir(), 'relay-sub-'))
|
||||
writeFileSync(path.join(tmpDir, 'test.txt'), 'hello')
|
||||
|
|
|
|||
Loading…
Reference in New Issue