From c419aadb190b2eb9c77d3d55dd79dd1a97d6fd08 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:30:31 -0700 Subject: [PATCH] =?UTF-8?q?oom(01):=20A1-shared-readers=20=E2=80=94=20rein?= =?UTF-8?q?troduce=20#10179=20subset=20(#10294)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Files: 18 applied, 0 deleted (from 6eb70d83703e5c69c2878caca73d536a2abbadab) Co-authored-by: Orca --- src/shared/bounded-secure-json-file.ts | 10 + src/shared/fetch-response-body.ts | 116 +++++++++ src/shared/growing-byte-buffer.test.ts | 54 +++++ src/shared/growing-byte-buffer.ts | 100 ++++++++ src/shared/node-file-content-equality.test.ts | 40 ++++ src/shared/node-file-content-equality.ts | 72 ++++++ src/shared/node-readable-text.test.ts | 40 ++++ src/shared/node-readable-text.ts | 42 ++++ .../node-source-copy-content-equality.test.ts | 57 +++++ .../node-source-copy-content-equality.ts | 64 +++++ src/shared/nul-delimited-fields.test.ts | 8 + src/shared/nul-delimited-fields.ts | 12 + .../persisted-state-file-bounds.test.ts | 171 ++++++++++++++ src/shared/persisted-state-file-bounds.ts | 222 ++++++++++++++++++ src/shared/search-subprocess-lines.test.ts | 55 +++++ src/shared/search-subprocess-lines.ts | 75 ++++++ src/shared/string-chunk-compaction.test.ts | 17 ++ src/shared/string-chunk-compaction.ts | 11 + 18 files changed, 1166 insertions(+) create mode 100644 src/shared/bounded-secure-json-file.ts create mode 100644 src/shared/fetch-response-body.ts create mode 100644 src/shared/growing-byte-buffer.test.ts create mode 100644 src/shared/growing-byte-buffer.ts create mode 100644 src/shared/node-file-content-equality.test.ts create mode 100644 src/shared/node-file-content-equality.ts create mode 100644 src/shared/node-readable-text.test.ts create mode 100644 src/shared/node-readable-text.ts create mode 100644 src/shared/node-source-copy-content-equality.test.ts create mode 100644 src/shared/node-source-copy-content-equality.ts create mode 100644 src/shared/nul-delimited-fields.test.ts create mode 100644 src/shared/nul-delimited-fields.ts create mode 100644 src/shared/persisted-state-file-bounds.test.ts create mode 100644 src/shared/persisted-state-file-bounds.ts create mode 100644 src/shared/search-subprocess-lines.test.ts create mode 100644 src/shared/search-subprocess-lines.ts create mode 100644 src/shared/string-chunk-compaction.test.ts create mode 100644 src/shared/string-chunk-compaction.ts diff --git a/src/shared/bounded-secure-json-file.ts b/src/shared/bounded-secure-json-file.ts new file mode 100644 index 000000000..c9da2b9ce --- /dev/null +++ b/src/shared/bounded-secure-json-file.ts @@ -0,0 +1,10 @@ +import { stringifyJsonWithinByteLimit } from './node-bounded-json-stringify' +import { writeSecureFile } from './secure-file' + +export function writeSecureJsonFileWithinLimit( + targetPath: string, + value: unknown, + maxBytes: number +): void { + writeSecureFile(targetPath, stringifyJsonWithinByteLimit(value, maxBytes).serialized) +} diff --git a/src/shared/fetch-response-body.ts b/src/shared/fetch-response-body.ts new file mode 100644 index 000000000..e9467fd3a --- /dev/null +++ b/src/shared/fetch-response-body.ts @@ -0,0 +1,116 @@ +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from './json-text-structure-limit' + +const INITIAL_RESPONSE_CAPACITY_BYTES = 64 * 1024 + +export const API_RESPONSE_MAX_BYTES = 16 * 1024 * 1024 +export const API_RESPONSE_JSON_LIMITS: JsonTextStructureLimits = { + structuralTokens: 1_000_000, + nestingDepth: 128 +} + +export class FetchResponseBodyTooLargeError extends Error { + constructor( + readonly observedBytes: number, + readonly maxBytes: number + ) { + super(`Response body exceeds ${maxBytes} byte limit`) + this.name = 'FetchResponseBodyTooLargeError' + } +} + +function parseContentLength(response: Response): number | null { + const raw = response.headers.get('content-length') + if (!raw || !/^\d+$/.test(raw)) { + return null + } + const parsed = Number(raw) + return Number.isSafeInteger(parsed) ? parsed : null +} + +function isHighLevelOnlyResponse(response: Response): boolean { + const partial = response as Partial + // Injected request adapters may expose only the high-level method they implement. + return partial.headers === undefined && partial.body === undefined +} + +async function cancelReader(reader: ReadableStreamDefaultReader): Promise { + try { + await reader.cancel() + } catch { + // An already-errored or closed response needs no further draining. + } +} + +export async function readFetchResponseBytesWithinLimit( + response: Response, + maxBytes = API_RESPONSE_MAX_BYTES +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Response body limit must be a non-negative safe integer') + } + + const contentLength = parseContentLength(response) + if (contentLength !== null && contentLength > maxBytes) { + await response.body?.cancel().catch(() => undefined) + throw new FetchResponseBodyTooLargeError(contentLength, maxBytes) + } + if (!response.body) { + return new Uint8Array() + } + + const reader = response.body.getReader() + let output = new Uint8Array(Math.min(maxBytes, INITIAL_RESPONSE_CAPACITY_BYTES)) + let byteLength = 0 + try { + while (true) { + const { done, value } = await reader.read() + if (done) { + return output.subarray(0, byteLength) + } + const nextLength = byteLength + value.byteLength + if (!Number.isSafeInteger(nextLength) || nextLength > maxBytes) { + await cancelReader(reader) + throw new FetchResponseBodyTooLargeError(nextLength, maxBytes) + } + if (nextLength > output.byteLength) { + const nextCapacity = Math.min( + maxBytes, + Math.max(INITIAL_RESPONSE_CAPACITY_BYTES, output.byteLength * 2, nextLength) + ) + const expanded = new Uint8Array(nextCapacity) + expanded.set(output.subarray(0, byteLength)) + output = expanded + } + output.set(value, byteLength) + byteLength = nextLength + } + } finally { + reader.releaseLock() + } +} + +export async function readFetchResponseTextWithinLimit( + response: Response, + maxBytes = API_RESPONSE_MAX_BYTES +): Promise { + if (isHighLevelOnlyResponse(response)) { + return response.text() + } + return new TextDecoder().decode(await readFetchResponseBytesWithinLimit(response, maxBytes)) +} + +export async function readFetchResponseJsonWithinLimit( + response: Response, + maxBytes = API_RESPONSE_MAX_BYTES, + structureLimits: JsonTextStructureLimits = API_RESPONSE_JSON_LIMITS +): Promise { + if (isHighLevelOnlyResponse(response)) { + return response.json() as Promise + } + const content = await readFetchResponseTextWithinLimit(response, maxBytes) + assertJsonTextStructureWithinLimits(content, structureLimits) + return JSON.parse(content) as T +} diff --git a/src/shared/growing-byte-buffer.test.ts b/src/shared/growing-byte-buffer.test.ts new file mode 100644 index 000000000..8dc477ae6 --- /dev/null +++ b/src/shared/growing-byte-buffer.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest' +import { GrowingByteBuffer } from './growing-byte-buffer' + +describe('GrowingByteBuffer', () => { + it('retains 100,000 one-byte fragments in one growable allocation', () => { + const buffer = new GrowingByteBuffer() + const expected = Buffer.alloc(100_000) + + for (let index = 0; index < expected.byteLength; index += 1) { + const value = index % 251 + expected[index] = value + buffer.append(Uint8Array.of(value)) + } + + expect(buffer.byteLength).toBe(expected.byteLength) + expect(buffer.takeString('latin1')).toBe(expected.toString('latin1')) + expect(buffer.byteLength).toBe(0) + }) + + it('consumes delimited prefixes and retains a bounded suffix', () => { + const buffer = new GrowingByteBuffer() + for (const byte of Buffer.from('first\nsecond-tail')) { + buffer.append(Uint8Array.of(byte)) + } + + const newline = buffer.indexOfByte(0x0a) + expect(buffer.takePrefixString(newline)).toBe('first') + buffer.discardPrefix(1) + buffer.retainSuffix(4) + + expect(buffer.toString()).toBe('tail') + }) + + it('appends only a bounded copy from an oversized source chunk', () => { + const buffer = new GrowingByteBuffer() + buffer.append(Buffer.from('old')) + const source = Buffer.from('discard-prefix-tail') + + buffer.appendRetainedSuffix(source, 4) + source.fill(0) + + expect(buffer.byteLength).toBe(4) + expect(buffer.toString()).toBe('tail') + }) + + it('keeps the newest bytes across bounded suffix appends', () => { + const buffer = new GrowingByteBuffer() + + buffer.appendRetainedSuffix(Buffer.from('1234'), 6) + buffer.appendRetainedSuffix(Buffer.from('5678'), 6) + + expect(buffer.toString()).toBe('345678') + }) +}) diff --git a/src/shared/growing-byte-buffer.ts b/src/shared/growing-byte-buffer.ts new file mode 100644 index 000000000..48c5d90de --- /dev/null +++ b/src/shared/growing-byte-buffer.ts @@ -0,0 +1,100 @@ +export class GrowingByteBuffer { + private storage = Buffer.alloc(0) + private length = 0 + + get byteLength(): number { + return this.length + } + + append(bytes: Buffer | Uint8Array): void { + if (bytes.byteLength === 0) { + return + } + const required = this.length + bytes.byteLength + if (required > this.storage.byteLength) { + const capacity = Math.max(required, Math.max(256, this.storage.byteLength * 2)) + const next = Buffer.allocUnsafe(capacity) + this.storage.copy(next, 0, 0, this.length) + this.storage = next + } + const source = Buffer.isBuffer(bytes) + ? bytes + : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) + source.copy(this.storage, this.length) + this.length = required + } + + appendRetainedSuffix(bytes: Buffer | Uint8Array, maxBytes: number): void { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Retained suffix limit must be a non-negative safe integer') + } + if (maxBytes === 0) { + this.clear() + return + } + const source = Buffer.isBuffer(bytes) + ? bytes + : Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) + if (source.byteLength >= maxBytes) { + this.storage = Buffer.from(source.subarray(source.byteLength - maxBytes)) + this.length = maxBytes + return + } + const retainedBytes = Math.min(this.length, maxBytes - source.byteLength) + if (retainedBytes < this.length) { + this.storage.copy(this.storage, 0, this.length - retainedBytes, this.length) + this.length = retainedBytes + } + this.append(source) + } + + indexOfByte(value: number, byteOffset = 0): number { + return this.storage.subarray(0, this.length).indexOf(value, byteOffset) + } + + takePrefixString(byteLength: number, encoding: BufferEncoding = 'utf8'): string { + if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > this.length) { + throw new RangeError('Prefix length exceeds retained bytes') + } + const value = this.storage.toString(encoding, 0, byteLength) + this.discardPrefix(byteLength) + return value + } + + discardPrefix(byteLength: number): void { + if (!Number.isSafeInteger(byteLength) || byteLength < 0 || byteLength > this.length) { + throw new RangeError('Prefix length exceeds retained bytes') + } + if (byteLength === 0) { + return + } + this.storage.copy(this.storage, 0, byteLength, this.length) + this.length -= byteLength + } + + retainSuffix(maxBytes: number): void { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Suffix limit must be a non-negative safe integer') + } + if (this.length <= maxBytes) { + return + } + this.storage.copy(this.storage, 0, this.length - maxBytes, this.length) + this.length = maxBytes + } + + toString(encoding: BufferEncoding = 'utf8'): string { + return this.storage.toString(encoding, 0, this.length) + } + + takeString(encoding: BufferEncoding = 'utf8'): string { + const value = this.toString(encoding) + this.clear() + return value + } + + clear(): void { + this.storage = Buffer.alloc(0) + this.length = 0 + } +} diff --git a/src/shared/node-file-content-equality.test.ts b/src/shared/node-file-content-equality.test.ts new file mode 100644 index 000000000..af50287b9 --- /dev/null +++ b/src/shared/node-file-content-equality.test.ts @@ -0,0 +1,40 @@ +import { mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { nodeFileContentsEqual, nodeFileContentsEqualSync } from './node-file-content-equality' + +const roots: string[] = [] + +function createFile(contents: string): string { + const root = mkdtempSync(join(tmpdir(), 'orca-file-content-equality-')) + roots.push(root) + const filePath = join(root, 'owned-launcher') + writeFileSync(filePath, contents) + return filePath +} + +afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } +}) + +describe('Node file content equality', () => { + it('compares UTF-8 content without changing its bytes', async () => { + const filePath = createFile('launch πŸ‹\n') + + await expect(nodeFileContentsEqual(filePath, 'launch πŸ‹\n')).resolves.toBe(true) + expect(nodeFileContentsEqualSync(filePath, 'launch πŸ‹\n')).toBe(true) + await expect(nodeFileContentsEqual(filePath, 'different\n')).resolves.toBe(false) + expect(nodeFileContentsEqualSync(filePath, 'different\n')).toBe(false) + }) + + it('rejects a large sparse replacement from metadata without reading its payload', async () => { + const filePath = createFile('owned launcher\n') + truncateSync(filePath, 256 * 1024 * 1024) + + await expect(nodeFileContentsEqual(filePath, 'owned launcher\n')).resolves.toBe(false) + expect(nodeFileContentsEqualSync(filePath, 'owned launcher\n')).toBe(false) + }) +}) diff --git a/src/shared/node-file-content-equality.ts b/src/shared/node-file-content-equality.ts new file mode 100644 index 000000000..5f7ccb729 --- /dev/null +++ b/src/shared/node-file-content-equality.ts @@ -0,0 +1,72 @@ +import { closeSync, fstatSync, openSync, readSync } from 'node:fs' +import { open } from 'node:fs/promises' + +export const NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES = 64 * 1024 + +function expectedBytes(contents: string | Buffer): Buffer { + return typeof contents === 'string' ? Buffer.from(contents, 'utf8') : contents +} + +export async function nodeFileContentsEqual( + filePath: string, + expectedContents: string | Buffer +): Promise { + const expected = expectedBytes(expectedContents) + const handle = await open(filePath, 'r') + try { + if ((await handle.stat()).size !== expected.length) { + return false + } + const chunk = Buffer.allocUnsafe( + Math.min(NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES, expected.length) + ) + let offset = 0 + while (offset < expected.length) { + const length = Math.min(chunk.length, expected.length - offset) + const { bytesRead } = await handle.read(chunk, 0, length, offset) + if ( + bytesRead === 0 || + !chunk.subarray(0, bytesRead).equals(expected.subarray(offset, offset + bytesRead)) + ) { + return false + } + offset += bytesRead + } + const probe = Buffer.allocUnsafe(1) + return (await handle.read(probe, 0, 1, offset)).bytesRead === 0 + } finally { + await handle.close() + } +} + +export function nodeFileContentsEqualSync( + filePath: string, + expectedContents: string | Buffer +): boolean { + const expected = expectedBytes(expectedContents) + const descriptor = openSync(filePath, 'r') + try { + if (fstatSync(descriptor).size !== expected.length) { + return false + } + const chunk = Buffer.allocUnsafe( + Math.min(NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES, expected.length) + ) + let offset = 0 + while (offset < expected.length) { + const length = Math.min(chunk.length, expected.length - offset) + const bytesRead = readSync(descriptor, chunk, 0, length, offset) + if ( + bytesRead === 0 || + !chunk.subarray(0, bytesRead).equals(expected.subarray(offset, offset + bytesRead)) + ) { + return false + } + offset += bytesRead + } + const probe = Buffer.allocUnsafe(1) + return readSync(descriptor, probe, 0, 1, offset) === 0 + } finally { + closeSync(descriptor) + } +} diff --git a/src/shared/node-readable-text.test.ts b/src/shared/node-readable-text.test.ts new file mode 100644 index 000000000..5205b589d --- /dev/null +++ b/src/shared/node-readable-text.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' +import { + NodeReadableTextTooLargeError, + readNodeReadableTextWithinLimit +} from './node-readable-text' + +async function* chunks(values: unknown[]): AsyncGenerator { + yield* values +} + +describe('readNodeReadableTextWithinLimit', () => { + it('preserves accepted UTF-8 bytes split across chunks', async () => { + const encoded = Buffer.from('hello 🌍') + + await expect( + readNodeReadableTextWithinLimit( + chunks([encoded.subarray(0, 8), encoded.subarray(8)]), + encoded.byteLength + ) + ).resolves.toBe('hello 🌍') + }) + + it('accepts input exactly at the byte limit', async () => { + await expect( + readNodeReadableTextWithinLimit(chunks(['ab', Buffer.from('cd')]), 4) + ).resolves.toBe('abcd') + }) + + it('rejects before retaining input beyond the byte limit', async () => { + await expect(readNodeReadableTextWithinLimit(chunks(['1234', '5']), 4)).rejects.toEqual( + new NodeReadableTextTooLargeError(5, 4) + ) + }) + + it('does not let an unlimited sequence of empty chunks grow retained state', async () => { + await expect( + readNodeReadableTextWithinLimit(chunks(Array.from({ length: 10_000 }, () => '')), 0) + ).resolves.toBe('') + }) +}) diff --git a/src/shared/node-readable-text.ts b/src/shared/node-readable-text.ts new file mode 100644 index 000000000..d73370629 --- /dev/null +++ b/src/shared/node-readable-text.ts @@ -0,0 +1,42 @@ +const INITIAL_READ_CAPACITY_BYTES = 64 * 1024 + +export class NodeReadableTextTooLargeError extends Error { + constructor( + readonly observedBytes: number, + readonly maxBytes: number + ) { + super(`Input exceeds ${maxBytes} byte limit (${observedBytes} bytes received)`) + this.name = 'NodeReadableTextTooLargeError' + } +} + +export async function readNodeReadableTextWithinLimit( + readable: AsyncIterable, + maxBytes: number +): Promise { + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Readable text limit must be a non-negative safe integer') + } + + let buffer = Buffer.allocUnsafe(Math.min(INITIAL_READ_CAPACITY_BYTES, maxBytes)) + let bytes = 0 + for await (const value of readable) { + const chunk = Buffer.isBuffer(value) ? value : Buffer.from(String(value)) + const observedBytes = bytes + chunk.byteLength + if (!Number.isSafeInteger(observedBytes) || observedBytes > maxBytes) { + throw new NodeReadableTextTooLargeError(observedBytes, maxBytes) + } + if (observedBytes > buffer.byteLength) { + const nextCapacity = Math.min( + maxBytes, + Math.max(observedBytes, INITIAL_READ_CAPACITY_BYTES, buffer.byteLength * 2) + ) + const expanded = Buffer.allocUnsafe(nextCapacity) + buffer.copy(expanded, 0, 0, bytes) + buffer = expanded + } + chunk.copy(buffer, bytes) + bytes = observedBytes + } + return buffer.subarray(0, bytes).toString('utf8') +} diff --git a/src/shared/node-source-copy-content-equality.test.ts b/src/shared/node-source-copy-content-equality.test.ts new file mode 100644 index 000000000..cebe42e99 --- /dev/null +++ b/src/shared/node-source-copy-content-equality.test.ts @@ -0,0 +1,57 @@ +import { + closeSync, + mkdtempSync, + openSync, + rmSync, + truncateSync, + writeFileSync, + writeSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import { + NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES, + nodeSourceAndCopyContentsEqualSync +} from './node-source-copy-content-equality' + +describe('Node source/copy content equality', () => { + const roots: string[] = [] + + afterEach(() => { + for (const root of roots.splice(0)) { + rmSync(root, { recursive: true, force: true }) + } + }) + + it('compares multi-megabyte sparse files with fixed-size chunks', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-resource-compare-')) + roots.push(root) + const sourcePath = join(root, 'source.md') + const copyPath = join(root, 'copy.md') + const sparseBytes = NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES * 128 + for (const path of [sourcePath, copyPath]) { + writeFileSync(path, 'same-prefix') + truncateSync(path, sparseBytes) + } + + expect(nodeSourceAndCopyContentsEqualSync(sourcePath, copyPath)).toBe(true) + + const descriptor = openSync(copyPath, 'r+') + try { + writeSync(descriptor, Buffer.from('x'), 0, 1, sparseBytes - 1) + } finally { + closeSync(descriptor) + } + expect(nodeSourceAndCopyContentsEqualSync(sourcePath, copyPath)).toBe(false) + }) + + it('rejects a non-file copy without attempting to consume it', () => { + const root = mkdtempSync(join(tmpdir(), 'orca-resource-compare-dir-')) + roots.push(root) + const sourcePath = join(root, 'source.md') + writeFileSync(sourcePath, 'contents') + + expect(nodeSourceAndCopyContentsEqualSync(sourcePath, root)).toBe(false) + }) +}) diff --git a/src/shared/node-source-copy-content-equality.ts b/src/shared/node-source-copy-content-equality.ts new file mode 100644 index 000000000..7c17a1eef --- /dev/null +++ b/src/shared/node-source-copy-content-equality.ts @@ -0,0 +1,64 @@ +import { closeSync, lstatSync, openSync, readSync, statSync } from 'node:fs' + +export const NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES = 64 * 1024 + +function readChunk(descriptor: number, buffer: Buffer): number { + let offset = 0 + while (offset < buffer.length) { + const bytesRead = readSync(descriptor, buffer, offset, buffer.length - offset, null) + if (bytesRead === 0) { + break + } + offset += bytesRead + } + return offset +} + +export function nodeSourceAndCopyContentsEqualSync(sourcePath: string, copyPath: string): boolean { + try { + // Why: source links are intentional, but an owned copy must remain a regular file. + if (!statSync(sourcePath).isFile() || !lstatSync(copyPath).isFile()) { + return false + } + } catch { + return false + } + + let sourceDescriptor: number | null = null + let copyDescriptor: number | null = null + let matches = false + let failed = false + try { + sourceDescriptor = openSync(sourcePath, 'r') + copyDescriptor = openSync(copyPath, 'r') + const sourceBuffer = Buffer.allocUnsafe(NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES) + const copyBuffer = Buffer.allocUnsafe(NODE_FILE_CONTENT_COMPARE_CHUNK_BYTES) + while (true) { + const sourceBytes = readChunk(sourceDescriptor, sourceBuffer) + const copyBytes = readChunk(copyDescriptor, copyBuffer) + if (sourceBytes !== copyBytes) { + break + } + if (sourceBytes === 0) { + matches = true + break + } + if (!sourceBuffer.subarray(0, sourceBytes).equals(copyBuffer.subarray(0, copyBytes))) { + break + } + } + } catch { + failed = true + } + for (const descriptor of [sourceDescriptor, copyDescriptor]) { + if (descriptor === null) { + continue + } + try { + closeSync(descriptor) + } catch { + failed = true + } + } + return matches && !failed +} diff --git a/src/shared/nul-delimited-fields.test.ts b/src/shared/nul-delimited-fields.test.ts new file mode 100644 index 000000000..59f8f751b --- /dev/null +++ b/src/shared/nul-delimited-fields.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, it } from 'vitest' +import { iterateNulDelimitedFields } from './nul-delimited-fields' + +describe('iterateNulDelimitedFields', () => { + it('preserves empty and trailing fields without materializing a split array', () => { + expect([...iterateNulDelimitedFields('one\0\0three\0')]).toEqual(['one', '', 'three', '']) + }) +}) diff --git a/src/shared/nul-delimited-fields.ts b/src/shared/nul-delimited-fields.ts new file mode 100644 index 000000000..64bbf04de --- /dev/null +++ b/src/shared/nul-delimited-fields.ts @@ -0,0 +1,12 @@ +export function* iterateNulDelimitedFields(value: string): Generator { + let start = 0 + while (start <= value.length) { + const end = value.indexOf('\0', start) + if (end === -1) { + yield value.slice(start) + return + } + yield value.slice(start, end) + start = end + 1 + } +} diff --git a/src/shared/persisted-state-file-bounds.test.ts b/src/shared/persisted-state-file-bounds.test.ts new file mode 100644 index 000000000..e6ee2de18 --- /dev/null +++ b/src/shared/persisted-state-file-bounds.test.ts @@ -0,0 +1,171 @@ +import { createHash } from 'node:crypto' +import { + mkdtempSync, + readFileSync, + readdirSync, + rmSync, + truncateSync, + writeFileSync +} from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { NodeFileReadTooLargeError } from './node-bounded-file-reader' +import { JsonStringifyByteLimitError } from './node-bounded-json-stringify' +import { + PersistedStateSecretCapacityError, + assertPersistedStateSecretWithinLimit, + readPersistedStateJsonFileSync, + replacePersistedStateJsonWithinLimit, + restorePersistedStateBackupSync, + stringifyPrettyPersistedStateWithinLimit, + stringifyPersistedStateWithinLimit, + updatePersistedStateHashWithJsonRange +} from './persisted-state-file-bounds' + +describe('persisted state file bounds', () => { + let root = '' + + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'orca-state-bounds-')) + }) + + afterEach(() => { + rmSync(root, { recursive: true, force: true }) + }) + + it('reads and parses a state file exactly at the byte limit', () => { + const path = join(root, 'state.json') + const json = `{"value":"${'x'.repeat(20)}"}` + writeFileSync(path, json) + + expect( + readPersistedStateJsonFileSync<{ value: string }>(path, Buffer.byteLength(json)) + ).toEqual({ + byteLength: Buffer.byteLength(json), + value: { value: 'x'.repeat(20) } + }) + }) + + it('rejects an oversized sparse state file before reading its body', () => { + const path = join(root, 'state.json') + writeFileSync(path, '') + truncateSync(path, 1025) + + expect(() => readPersistedStateJsonFileSync(path, 1024)).toThrow(NodeFileReadTooLargeError) + }) + + it('rejects structurally amplified state before parsing it', () => { + const path = join(root, 'state.json') + const json = '{"rows":[{},{}]}' + writeFileSync(path, json) + + expect(() => + readPersistedStateJsonFileSync(path, Buffer.byteLength(json), { + structuralTokens: 7, + nestingDepth: 3 + }) + ).toThrow('JSON structure') + }) + + it('matches native compact JSON exactly at the output boundary', () => { + const state = { quote: '"', unicode: 'πŸ‹', nested: [1, true, null] } + const native = JSON.stringify(state) + + expect(stringifyPersistedStateWithinLimit(state, Buffer.byteLength(native))).toEqual({ + byteLength: Buffer.byteLength(native), + serialized: native + }) + expect(() => stringifyPersistedStateWithinLimit(state, Buffer.byteLength(native) - 1)).toThrow( + JsonStringifyByteLimitError + ) + }) + + it('matches native pretty JSON and enforces its whitespace-inclusive boundary', () => { + const state = { nested: { value: 'x' }, list: [1, 2] } + const native = JSON.stringify(state, null, 2) + + expect(stringifyPrettyPersistedStateWithinLimit(state, Buffer.byteLength(native))).toEqual({ + byteLength: Buffer.byteLength(native), + serialized: native + }) + expect(() => + stringifyPrettyPersistedStateWithinLimit(state, Buffer.byteLength(native) - 1) + ).toThrow(JsonStringifyByteLimitError) + }) + + it('bounds secret plaintext before encryption can expand it', () => { + assertPersistedStateSecretWithinLimit('πŸ‹', 4) + + expect(() => assertPersistedStateSecretWithinLimit('πŸ‹x', 4)).toThrow( + PersistedStateSecretCapacityError + ) + }) + + it('checks replacement growth before constructing the next payload', () => { + const serialized = '{"value":"slot"}' + const exactBytes = + Buffer.byteLength(serialized) - Buffer.byteLength('slot') + Buffer.byteLength('expanded') + + expect( + replacePersistedStateJsonWithinLimit({ + serialized, + currentBytes: Buffer.byteLength(serialized), + search: 'slot', + replacement: 'expanded', + maxBytes: exactBytes + }) + ).toEqual({ byteLength: exactBytes, serialized: '{"value":"expanded"}' }) + expect(() => + replacePersistedStateJsonWithinLimit({ + serialized, + currentBytes: Buffer.byteLength(serialized), + search: 'slot', + replacement: 'expanded', + maxBytes: exactBytes - 1 + }) + ).toThrow(JsonStringifyByteLimitError) + }) + + it('hashes bounded string ranges without splitting UTF-16 surrogate pairs', () => { + const value = `prefix-${'x'.repeat(8)}πŸ‹-${'y'.repeat(8)}-suffix` + const expected = createHash('sha1').update(value).digest('hex') + const actual = createHash('sha1') + + updatePersistedStateHashWithJsonRange(actual, value, 0, value.length, 2) + + expect(actual.digest('hex')).toBe(expected) + }) + + it('atomically restores only a valid in-limit backup', () => { + const backupPath = join(root, 'backup.json') + const targetPath = join(root, 'profile', 'orca-data.json') + writeFileSync(backupPath, '{"repos":[{"id":"recovered"}]}') + + restorePersistedStateBackupSync(backupPath, targetPath, 1024) + + expect(JSON.parse(readFileSync(targetPath, 'utf8'))).toEqual({ + repos: [{ id: 'recovered' }] + }) + const originalTarget = readFileSync(targetPath) + writeFileSync(backupPath, '{{invalid') + expect(() => restorePersistedStateBackupSync(backupPath, targetPath, 1024)).toThrow() + expect(readFileSync(targetPath)).toEqual(originalTarget) + expect( + readdirSync(join(root, 'profile')).filter((name) => name.endsWith('.recovery.tmp')) + ).toEqual([]) + }) + + it('leaves the target untouched when a backup exceeds the cap', () => { + const backupPath = join(root, 'backup.json') + const targetPath = join(root, 'orca-data.json') + writeFileSync(targetPath, '{"original":true}') + writeFileSync(backupPath, '') + truncateSync(backupPath, 1025) + + expect(() => restorePersistedStateBackupSync(backupPath, targetPath, 1024)).toThrow( + NodeFileReadTooLargeError + ) + expect(readFileSync(targetPath, 'utf8')).toBe('{"original":true}') + }) +}) diff --git a/src/shared/persisted-state-file-bounds.ts b/src/shared/persisted-state-file-bounds.ts new file mode 100644 index 000000000..4fe8a977b --- /dev/null +++ b/src/shared/persisted-state-file-bounds.ts @@ -0,0 +1,222 @@ +import { randomUUID, type Hash } from 'node:crypto' +import { mkdirSync, renameSync, rmSync, writeFileSync } from 'node:fs' +import { dirname } from 'node:path' +import { + NodeFileReadTooLargeError, + readNodeFileSyncWithinLimit, + type BoundedNodeFileRead +} from './node-bounded-file-reader' +import { + JsonStringifyByteLimitError, + stringifyJsonWithinByteLimit +} from './node-bounded-json-stringify' +import { + assertJsonTextStructureWithinLimits, + type JsonTextStructureLimits +} from './json-text-structure-limit' + +export const ORCA_PERSISTED_STATE_MAX_BYTES = 64 * 1024 * 1024 +export const ORCA_PERSISTED_STATE_SECRET_MAX_BYTES = 4 * 1024 * 1024 +export const ORCA_PERSISTED_STATE_HASH_CHUNK_CODE_UNITS = 64 * 1024 +export const ORCA_PERSISTED_STATE_JSON_LIMITS: JsonTextStructureLimits = { + structuralTokens: 4_000_000, + nestingDepth: 256 +} + +export type PersistedStateJsonRead = { + byteLength: number + value: T +} + +export class PersistedStateSecretCapacityError extends Error { + constructor( + readonly observedBytes: number, + readonly maxBytes = ORCA_PERSISTED_STATE_SECRET_MAX_BYTES + ) { + super(`Persisted state secret exceeds ${maxBytes} bytes`) + this.name = 'PersistedStateSecretCapacityError' + } +} + +export function isPersistedStateFileCapacityError( + error: unknown +): error is NodeFileReadTooLargeError { + return error instanceof NodeFileReadTooLargeError +} + +export function readPersistedStateJsonFileSync( + filePath: string, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES, + structureLimits: JsonTextStructureLimits = ORCA_PERSISTED_STATE_JSON_LIMITS +): PersistedStateJsonRead { + const { buffer } = readPersistedStateFileBytesSync(filePath, maxBytes) + return { + byteLength: buffer.byteLength, + value: parsePersistedStateJsonBuffer(buffer, structureLimits) + } +} + +export function readPersistedStateFileBytesSync( + filePath: string, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES +): BoundedNodeFileRead { + return readNodeFileSyncWithinLimit(filePath, maxBytes) +} + +export function parsePersistedStateJsonBuffer( + buffer: Buffer, + structureLimits: JsonTextStructureLimits = ORCA_PERSISTED_STATE_JSON_LIMITS +): T { + const serialized = buffer.toString('utf8') + assertJsonTextStructureWithinLimits(serialized, structureLimits) + return JSON.parse(serialized) as T +} + +export function stringifyPersistedStateWithinLimit( + value: unknown, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES +): { byteLength: number; serialized: string } { + return stringifyJsonWithinByteLimit(value, maxBytes) +} + +export function stringifyPrettyPersistedStateWithinLimit( + value: unknown, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES +): { byteLength: number; serialized: string } { + return stringifyJsonWithinByteLimit(value, maxBytes, 2) +} + +export function encodePersistedStateJsonStringContent( + value: string, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES +): string { + const { serialized } = stringifyJsonWithinByteLimit(value, maxBytes) + return serialized.slice(1, -1) +} + +export function assertPersistedStateSecretWithinLimit( + value: string, + maxBytes = ORCA_PERSISTED_STATE_SECRET_MAX_BYTES +): void { + const observedBytes = Buffer.byteLength(value, 'utf8') + if (observedBytes > maxBytes) { + throw new PersistedStateSecretCapacityError(observedBytes, maxBytes) + } +} + +export function replacedPersistedStateJsonByteLength(options: { + currentBytes: number + maxBytes?: number + replacement: string + search: string +}): number { + const maxBytes = options.maxBytes ?? ORCA_PERSISTED_STATE_MAX_BYTES + if (!Number.isSafeInteger(maxBytes) || maxBytes < 0) { + throw new RangeError('Persisted state JSON byte limit must be a non-negative safe integer') + } + if (!Number.isSafeInteger(options.currentBytes) || options.currentBytes < 0) { + throw new RangeError('Persisted state JSON byte count must be a non-negative safe integer') + } + if (options.currentBytes > maxBytes) { + throw new JsonStringifyByteLimitError(options.currentBytes, maxBytes) + } + const nextBytes = + options.currentBytes - + Buffer.byteLength(options.search, 'utf8') + + Buffer.byteLength(options.replacement, 'utf8') + if (!Number.isSafeInteger(nextBytes) || nextBytes < 0 || nextBytes > maxBytes) { + throw new JsonStringifyByteLimitError(nextBytes, maxBytes) + } + return nextBytes +} + +export function replacePersistedStateJsonWithinLimit(options: { + currentBytes: number + maxBytes?: number + replacement: string + search: string + serialized: string +}): { byteLength: number; serialized: string } { + const byteLength = replacedPersistedStateJsonByteLength(options) + const searchIndex = options.serialized.indexOf(options.search) + if (searchIndex === -1) { + throw new Error('Persisted state JSON replacement slot is missing') + } + if (options.serialized.includes(options.search, searchIndex + options.search.length)) { + throw new Error('Persisted state JSON replacement slot is ambiguous') + } + return { + byteLength, + serialized: options.serialized.replace(options.search, () => options.replacement) + } +} + +export function updatePersistedStateHashWithJsonRange( + hash: Pick, + value: string, + start = 0, + end = value.length, + chunkCodeUnits = ORCA_PERSISTED_STATE_HASH_CHUNK_CODE_UNITS +): void { + if ( + !Number.isSafeInteger(start) || + !Number.isSafeInteger(end) || + start < 0 || + end < start || + end > value.length + ) { + throw new RangeError('Persisted state hash range is invalid') + } + if (!Number.isSafeInteger(chunkCodeUnits) || chunkCodeUnits <= 0) { + throw new RangeError('Persisted state hash chunk size must be a positive safe integer') + } + + let offset = start + while (offset < end) { + let nextOffset = Math.min(end, offset + chunkCodeUnits) + if ( + nextOffset < end && + isHighSurrogate(value.charCodeAt(nextOffset - 1)) && + isLowSurrogate(value.charCodeAt(nextOffset)) + ) { + nextOffset += 1 + } + hash.update(value.slice(offset, nextOffset), 'utf8') + offset = nextOffset + } +} + +export function restorePersistedStateBackupSync( + sourcePath: string, + targetPath: string, + maxBytes = ORCA_PERSISTED_STATE_MAX_BYTES +): number { + const read = readValidatedPersistedStateBytesSync(sourcePath, maxBytes) + mkdirSync(dirname(targetPath), { recursive: true }) + const temporaryPath = `${targetPath}.${process.pid}.${randomUUID()}.recovery.tmp` + try { + writeFileSync(temporaryPath, read.buffer) + renameSync(temporaryPath, targetPath) + } catch (error) { + rmSync(temporaryPath, { force: true }) + throw error + } + return read.buffer.byteLength +} + +function readValidatedPersistedStateBytesSync( + filePath: string, + maxBytes: number +): BoundedNodeFileRead { + const read = readPersistedStateFileBytesSync(filePath, maxBytes) + parsePersistedStateJsonBuffer(read.buffer) + return read +} + +function isHighSurrogate(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff +} + +function isLowSurrogate(code: number): boolean { + return code >= 0xdc00 && code <= 0xdfff +} diff --git a/src/shared/search-subprocess-lines.test.ts b/src/shared/search-subprocess-lines.test.ts new file mode 100644 index 000000000..334477607 --- /dev/null +++ b/src/shared/search-subprocess-lines.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest' +import { SearchSubprocessLineAccumulator } from './search-subprocess-lines' + +describe('SearchSubprocessLineAccumulator', () => { + it('preserves UTF-8 records split across raw byte chunks', () => { + const parser = new SearchSubprocessLineAccumulator(32) + const bytes = Buffer.from('firstπŸ‹\nsecond') + const lines: string[] = [] + + expect(parser.push(bytes.subarray(0, 7), (line) => lines.push(line))).toBe(true) + expect(parser.push(bytes.subarray(7), (line) => lines.push(line))).toBe(true) + + expect(lines).toEqual(['firstπŸ‹']) + expect(parser.finish()).toBe('second') + }) + + it('accepts an exact byte limit and rejects the next byte without decoding it', () => { + const parser = new SearchSubprocessLineAccumulator(4) + const lines: string[] = [] + + expect(parser.push(Buffer.from('four\n'), (line) => lines.push(line))).toBe(true) + expect(parser.push(Buffer.from('fives'), (line) => lines.push(line))).toBe(false) + + expect(lines).toEqual(['four']) + expect(parser.finish()).toBeNull() + }) + + it('preserves empty lines and line order within one chunk', () => { + const parser = new SearchSubprocessLineAccumulator(8) + const lines: string[] = [] + + expect(parser.push(Buffer.from('\na\n\n'), (line) => lines.push(line))).toBe(true) + + expect(lines).toEqual(['', 'a', '']) + }) + + it('retains one growable buffer for adversarial one-byte fragments', () => { + const parser = new SearchSubprocessLineAccumulator(256 * 1024) + const byte = Buffer.from('x') + let accepted = true + + for (let index = 0; index < 200_000; index += 1) { + accepted = parser.push(byte, () => {}) && accepted + } + + expect(accepted).toBe(true) + expect(Reflect.get(parser, 'buffer')).toBeInstanceOf(Buffer) + expect(parser.finish()).toBe('x'.repeat(200_000)) + expect(Reflect.get(parser, 'buffer')).toBeNull() + }) + + it('rejects invalid byte limits', () => { + expect(() => new SearchSubprocessLineAccumulator(-1)).toThrow(RangeError) + }) +}) diff --git a/src/shared/search-subprocess-lines.ts b/src/shared/search-subprocess-lines.ts new file mode 100644 index 000000000..5c04d9e82 --- /dev/null +++ b/src/shared/search-subprocess-lines.ts @@ -0,0 +1,75 @@ +export const SEARCH_SUBPROCESS_MAX_LINE_BYTES = 64 * 1024 * 1024 +const SEARCH_SUBPROCESS_INITIAL_LINE_BUFFER_BYTES = 4 * 1024 + +export class SearchSubprocessLineAccumulator { + private buffer: Buffer | null = null + private bytes = 0 + + constructor(private readonly maxLineBytes = SEARCH_SUBPROCESS_MAX_LINE_BYTES) { + if (!Number.isSafeInteger(maxLineBytes) || maxLineBytes < 0) { + throw new RangeError('Search line limit must be a non-negative safe integer') + } + } + + push(rawChunk: Buffer | string, onLine: (line: string) => void): boolean { + const chunk = Buffer.isBuffer(rawChunk) ? rawChunk : Buffer.from(rawChunk, 'utf8') + let cursor = 0 + while (cursor < chunk.length) { + const newline = chunk.indexOf(0x0a, cursor) + const end = newline === -1 ? chunk.length : newline + const segmentBytes = end - cursor + if (this.bytes + segmentBytes > this.maxLineBytes) { + this.clear() + return false + } + + if (newline !== -1 && this.bytes === 0) { + onLine(chunk.toString('utf8', cursor, end)) + } else if (segmentBytes > 0) { + this.append(chunk.subarray(cursor, end)) + if (newline !== -1) { + onLine(this.takeLine()) + } + } else if (newline !== -1) { + onLine(this.takeLine()) + } + + if (newline === -1) { + return true + } + cursor = newline + 1 + } + return true + } + + finish(): string | null { + return this.bytes > 0 ? this.takeLine() : null + } + + clear(): void { + this.buffer = null + this.bytes = 0 + } + + private append(segment: Buffer): void { + const requiredBytes = this.bytes + segment.length + if (!this.buffer || this.buffer.length < requiredBytes) { + const doubledCapacity = this.buffer?.length ? this.buffer.length * 2 : 0 + const nextCapacity = Math.min( + this.maxLineBytes, + Math.max(SEARCH_SUBPROCESS_INITIAL_LINE_BUFFER_BYTES, doubledCapacity, requiredBytes) + ) + const next = Buffer.allocUnsafe(nextCapacity) + this.buffer?.copy(next, 0, 0, this.bytes) + this.buffer = next + } + segment.copy(this.buffer, this.bytes) + this.bytes = requiredBytes + } + + private takeLine(): string { + const line = this.buffer?.toString('utf8', 0, this.bytes) ?? '' + this.clear() + return line + } +} diff --git a/src/shared/string-chunk-compaction.test.ts b/src/shared/string-chunk-compaction.test.ts new file mode 100644 index 000000000..dad53ba1d --- /dev/null +++ b/src/shared/string-chunk-compaction.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from 'vitest' +import { appendCompactedStringChunk, RETAINED_STRING_CHUNK_LIMIT } from './string-chunk-compaction' + +describe('appendCompactedStringChunk', () => { + it('preserves 100,000 fragments within the retained chunk limit', () => { + const chunks: string[] = [] + let maxRetainedChunks = 0 + + for (let index = 0; index < 100_000; index += 1) { + appendCompactedStringChunk(chunks, String.fromCharCode(97 + (index % 26))) + maxRetainedChunks = Math.max(maxRetainedChunks, chunks.length) + } + + expect(maxRetainedChunks).toBeLessThanOrEqual(RETAINED_STRING_CHUNK_LIMIT) + expect(chunks.join('')).toHaveLength(100_000) + }) +}) diff --git a/src/shared/string-chunk-compaction.ts b/src/shared/string-chunk-compaction.ts new file mode 100644 index 000000000..1acb98ead --- /dev/null +++ b/src/shared/string-chunk-compaction.ts @@ -0,0 +1,11 @@ +export const RETAINED_STRING_CHUNK_LIMIT = 1_024 + +export function appendCompactedStringChunk(chunks: string[], value: string): void { + chunks.push(value) + if (chunks.length <= RETAINED_STRING_CHUNK_LIMIT) { + return + } + const compacted = chunks.join('') + chunks.length = 0 + chunks.push(compacted) +}