diff --git a/src/main/speech/model-catalog.test.ts b/src/main/speech/model-catalog.test.ts index 9dc099163..f81f57bbf 100644 --- a/src/main/speech/model-catalog.test.ts +++ b/src/main/speech/model-catalog.test.ts @@ -12,14 +12,11 @@ describe('SPEECH_MODEL_CATALOG', () => { expect(manifest?.streaming).toBe(false) expect(manifest?.sampleRate).toBe(16000) expect(manifest?.files).toEqual(['model.int8.onnx', 'tokens.txt']) - expect(manifest?.archiveFormat).toBe('tar.bz2') - expect(manifest?.sizeBytes).toBe(489_389_564) - expect(manifest?.downloadUrl).toBe( - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2' - ) - expect(manifest?.archiveSha256).toBe( - '4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b' - ) + expect(manifest?.sizeBytes).toBe(655_571_161) + expect(manifest?.downloadFiles?.map(({ name }) => name)).toEqual([ + 'model.int8.onnx', + 'tokens.txt' + ]) }) it('has unique ids across the catalog', () => { @@ -42,14 +39,10 @@ describe('SPEECH_MODEL_CATALOG', () => { expect(model?.files).toEqual(['model.int8.onnx', 'tokens.txt']) }) - it('downloads the upstream int8-only SenseVoice archive', () => { + it('downloads only the pinned SenseVoice runtime files', () => { const model = getCatalogModel('sense-voice-zh-en-ja-ko-yue') - expect(model?.sizeBytes).toBe(163_002_883) - expect(model?.downloadUrl).toBe( - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2' - ) - expect(model?.archiveSha256).toBe( - '7d1efa2138a65b0b488df37f8b89e3d91a60676e416f515b952358d83dfd347e' - ) + expect(model?.sizeBytes).toBe(239_549_735) + expect(model?.downloadFiles).toHaveLength(2) + expect(model?.downloadFiles?.map(({ name }) => name)).toEqual(['model.int8.onnx', 'tokens.txt']) }) }) diff --git a/src/main/speech/model-catalog.ts b/src/main/speech/model-catalog.ts index 65c74d18b..670999b44 100644 --- a/src/main/speech/model-catalog.ts +++ b/src/main/speech/model-catalog.ts @@ -1,7 +1,6 @@ import type { SpeechModelManifest } from '../../shared/speech-types' +import { getSpeechModelDownloadMetadata } from './model-download-catalog' -// Why: sizeBytes must be the exact upstream asset size — it is the UI size -// label and the download-progress denominator when content-length is missing. export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ { id: 'parakeet-tdt-0.6b-v3-int8', @@ -11,12 +10,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'multilingual', - sizeBytes: 487_170_055, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2', - archiveSha256: '5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf', - archiveFormat: 'tar.bz2', - files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt'], + ...getSpeechModelDownloadMetadata('parakeet-tdt-0.6b-v3-int8'), sampleRate: 16000, streaming: false, modelingUnit: 'bpe', @@ -30,12 +24,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'en', - sizeBytes: 482_468_385, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2', - archiveSha256: '157c157bc51155e03e37d2466522a3a737dd9c72bb25f36eb18912964161e1ad', - archiveFormat: 'tar.bz2', - files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt'], + ...getSpeechModelDownloadMetadata('parakeet-tdt-0.6b-v2-int8'), sampleRate: 16000, streaming: false, modelingUnit: 'bpe' @@ -47,17 +36,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'zh-en', - sizeBytes: 511_274_346, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2', - archiveSha256: '27ffbd9ee24ad186d99acc2f6354d7992b27bcab490812510665fa8f9389c5f8', - archiveFormat: 'tar.bz2', - files: [ - 'encoder-epoch-99-avg-1.onnx', - 'decoder-epoch-99-avg-1.onnx', - 'joiner-epoch-99-avg-1.onnx', - 'tokens.txt' - ], + ...getSpeechModelDownloadMetadata('zipformer-bilingual-zh-en'), sampleRate: 16000, streaming: true, modelingUnit: 'cjkchar+bpe' @@ -70,12 +49,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'paraformer', provider: 'local', language: 'zh-en', - sizeBytes: 1_047_319_737, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2', - archiveSha256: '5462a1fce42693deae572af1e8c4687124b12aa85fe61ff4d3168bb5280e205f', - archiveFormat: 'tar.bz2', - files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'tokens.txt'], + ...getSpeechModelDownloadMetadata('paraformer-bilingual-zh-en'), sampleRate: 16000, streaming: true }, @@ -86,17 +60,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'en', - sizeBytes: 127_887_156, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2', - archiveSha256: '9c559283e8498d3fe95913c79ca1cb454bb26281ac2b102b41306c7d752765d9', - archiveFormat: 'tar.bz2', - files: [ - 'encoder-epoch-99-avg-1.onnx', - 'decoder-epoch-99-avg-1.onnx', - 'joiner-epoch-99-avg-1.onnx', - 'tokens.txt' - ], + ...getSpeechModelDownloadMetadata('zipformer-streaming-en-20m'), sampleRate: 16000, streaming: true, modelingUnit: 'bpe' @@ -108,17 +72,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'zh', - sizeBytes: 74_004_050, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2', - archiveSha256: '2cbd71b640d9c37d3784f29367333a4577b0398b62e9deeed418170b081cba8b', - archiveFormat: 'tar.bz2', - files: [ - 'encoder-epoch-99-avg-1.onnx', - 'decoder-epoch-99-avg-1.onnx', - 'joiner-epoch-99-avg-1.onnx', - 'tokens.txt' - ], + ...getSpeechModelDownloadMetadata('zipformer-streaming-zh-14m'), sampleRate: 16000, streaming: true, modelingUnit: 'cjkchar' @@ -130,17 +84,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'transducer', provider: 'local', language: 'ko', - sizeBytes: 418_218_652, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2', - archiveSha256: 'e346a5882a409650472be17326237e24df7bf409db6b4a8a52e1a61422bf2500', - archiveFormat: 'tar.bz2', - files: [ - 'encoder-epoch-99-avg-1.int8.onnx', - 'decoder-epoch-99-avg-1.int8.onnx', - 'joiner-epoch-99-avg-1.int8.onnx', - 'tokens.txt' - ], + ...getSpeechModelDownloadMetadata('zipformer-streaming-korean'), sampleRate: 16000, streaming: true, modelingUnit: 'bpe' @@ -152,12 +96,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'nemo-ctc', provider: 'local', language: 'ja', - sizeBytes: 489_389_564, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2', - archiveSha256: '4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b', - archiveFormat: 'tar.bz2', - files: ['model.int8.onnx', 'tokens.txt'], + ...getSpeechModelDownloadMetadata('parakeet-tdt-ctc-0.6b-ja-int8'), sampleRate: 16000, streaming: false }, @@ -168,12 +107,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'whisper', provider: 'local', language: 'multilingual', - sizeBytes: 116_204_861, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2', - archiveSha256: 'c46116994e539aa165266d96b325252728429c12535eb9d8b6a2b10f129e66b1', - archiveFormat: 'tar.bz2', - files: ['tiny-encoder.onnx', 'tiny-decoder.onnx', 'tiny-tokens.txt'], + ...getSpeechModelDownloadMetadata('whisper-tiny'), sampleRate: 16000, streaming: false }, @@ -185,12 +119,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [ type: 'senseVoice', provider: 'local', language: 'multilingual', - sizeBytes: 163_002_883, - downloadUrl: - 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2', - archiveSha256: '7d1efa2138a65b0b488df37f8b89e3d91a60676e416f515b952358d83dfd347e', - archiveFormat: 'tar.bz2', - files: ['model.int8.onnx', 'tokens.txt'], + ...getSpeechModelDownloadMetadata('sense-voice-zh-en-ja-ko-yue'), sampleRate: 16000, streaming: false }, diff --git a/src/main/speech/model-download-catalog.ts b/src/main/speech/model-download-catalog.ts new file mode 100644 index 000000000..e71e4966c --- /dev/null +++ b/src/main/speech/model-download-catalog.ts @@ -0,0 +1,230 @@ +import type { SpeechModelDownloadFile } from '../../shared/speech-types' + +type DownloadFileSpec = readonly [name: string, sizeBytes: number, sha256: string] + +function huggingFaceFiles( + repository: string, + revision: string, + specs: DownloadFileSpec[] +): SpeechModelDownloadFile[] { + return specs.map(([name, sizeBytes, sha256]) => ({ + name, + url: `https://huggingface.co/${repository}/resolve/${revision}/${encodeURIComponent(name)}?download=true`, + sizeBytes, + sha256 + })) +} + +// Why: immutable revisions plus per-file hashes keep direct downloads equivalent to pinned archives. +const MODEL_DOWNLOAD_FILES = { + 'parakeet-tdt-0.6b-v3-int8': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8', + '2bda32ec70b097a55adaa07d9a7173915b43cc78', + [ + [ + 'encoder.int8.onnx', + 652_184_281, + 'acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247' + ], + [ + 'decoder.int8.onnx', + 11_845_275, + '179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e' + ], + [ + 'joiner.int8.onnx', + 6_355_277, + '3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3' + ], + ['tokens.txt', 93_939, 'd58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d'] + ] + ), + 'parakeet-tdt-0.6b-v2-int8': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8', + '1ab9323565ddb038682214b292f588070a538ce2', + [ + [ + 'encoder.int8.onnx', + 652_184_296, + 'a32b12d17bbbc309d0686fbbcc2987b5e9b8333a7da83fa6b089f0a2acd651ab' + ], + [ + 'decoder.int8.onnx', + 7_257_753, + 'b6bb64963457237b900e496ee9994b59294526439fbcc1fecf705b31a15c6b4e' + ], + [ + 'joiner.int8.onnx', + 1_739_080, + '7946164367946e7f9f29a122407c3252b680dbae9a51343eb2488d057c3c43d2' + ], + ['tokens.txt', 9_384, 'ec182b70dd42113aff6c5372c75cac58c952443eb22322f57bbd7f53977d497d'] + ] + ), + 'zipformer-bilingual-zh-en': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20', + '98590b7ed6443e77b714204da2757d75e1a642f4', + [ + [ + 'encoder-epoch-99-avg-1.onnx', + 330_083_505, + '709f0ed53a734b7942f170127e7547b566cb29c4afc5e67719f314c3d63ccb10' + ], + [ + 'decoder-epoch-99-avg-1.onnx', + 13_876_452, + '2e3b5ec371f8899ee6acd829fd753ba45772df57a91bdf37cde3136354e7db7d' + ], + [ + 'joiner-epoch-99-avg-1.onnx', + 12_833_618, + '5f2adc585dd1bec6421c8bb8660d2a73fc8b9ceb24491ef51399ba2a2f0fc31b' + ], + ['tokens.txt', 56_317, 'a8e0e4ec53810e433789b54a5c0134a7eaa2ffca595a6334d54c00da858841d3'], + ['bpe.vocab', 12_564, 'd0b642f3a2eacd5fadefdeff9e0e1358cab729647cbb7fe58cf738e1f7407029'] + ] + ), + 'paraformer-bilingual-zh-en': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en', + '8e40c43232a1c5c66c82111efc5820d3accca11b', + [ + [ + 'encoder.int8.onnx', + 165_462_184, + '81a70226a8934e6ed92aa1d4fc486b428b5398e2f2619ed4897b7294cab90e9a' + ], + [ + 'decoder.int8.onnx', + 71_664_561, + 'f3cca9f77bb9d93c8fcbfb63ae617b6b1ee96818df3aa3b151c40658fe38594f' + ], + ['tokens.txt', 75_756, '59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6'] + ] + ), + 'zipformer-streaming-en-20m': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17', + 'd42f2d9f7ca24806fb667456a18a9f1b60f70d16', + [ + [ + 'encoder-epoch-99-avg-1.onnx', + 88_804_590, + 'f77a22f4ff94604e1afb2aeb13504d7699363528c047c97d3436087c95c9b659' + ], + [ + 'decoder-epoch-99-avg-1.onnx', + 2_092_272, + '45a7f940ecfb53d89fa270ad11b88b961e53a317203eb24b1c8e95ed208b0f30' + ], + [ + 'joiner-epoch-99-avg-1.onnx', + 1_026_462, + '343e17dffa4f386ca206e00d3c406908f68f473c3d35968d6c3cddd5b8559a94' + ], + ['tokens.txt', 5_048, '49e3c2646595fd907228b3c6787069658f67b17377c60aeb8619c4551b2316fb'] + ] + ), + 'zipformer-streaming-zh-14m': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23', + '204ad334e2e683fd295359930cc16fc0432a23ac', + [ + [ + 'encoder-epoch-99-avg-1.onnx', + 40_948_171, + '84c6a8f372686faa5b8f45f2d79f0816f76dcd9f547acb9a90eba2772d7eda8b' + ], + [ + 'decoder-epoch-99-avg-1.onnx', + 7_509_745, + '5ee0f03a2768ff1d5c83ef3a493243c7935d316cd41280037b14783a3467cc78' + ], + [ + 'joiner-epoch-99-avg-1.onnx', + 7_109_975, + '030212efaea9a8b6a4fa98faf6ac6055529c4408cf4865e898220ddd02780f34' + ], + ['tokens.txt', 48_697, '8b294db9045d6e5f94647f4c1eec1af4da143a75053c399611444b378ff966ac'] + ] + ), + 'zipformer-streaming-korean': huggingFaceFiles( + 'k2-fsa/sherpa-onnx-streaming-zipformer-korean-2024-06-16', + 'ba6078bca4daf3f0dd37f79d0ab505af71df14a6', + [ + [ + 'encoder-epoch-99-avg-1.int8.onnx', + 126_968_852, + '8d0b1aa24fbedd4e3948564ab7facd151b8ce9b0c48fc987c541de2de3af5697' + ], + [ + 'decoder-epoch-99-avg-1.int8.onnx', + 2_844_692, + '68ea197936aabd249f38b53a87c775422bca64428ad4427d0e6e8092593e71fb' + ], + [ + 'joiner-epoch-99-avg-1.int8.onnx', + 2_581_421, + '128b80a66a1f718488af8560f9d15895109b99ff3e573f0a0130e03774ef1ced' + ], + ['tokens.txt', 60_246, '016bdf0965029263b7ad01b742366ee542ef0bef38261510e8176ff6f2e9e668'] + ] + ), + 'parakeet-tdt-ctc-0.6b-ja-int8': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8', + 'bef18eb066808c90bd0f5df5be685767b0732de8', + [ + [ + 'model.int8.onnx', + 655_542_604, + '3addd00ef5bd1742078389e540b77394e4a508bdf2f4c9ad1b4a76d93e76598e' + ], + ['tokens.txt', 28_557, '732f64c53909f2620c713f4106b487d92e6f54a6915b3cd3d1dbd32f9f4f392a'] + ] + ), + 'whisper-tiny': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-whisper-tiny', + '65176e2deb88badc814a94058666cadccc29b61c', + [ + [ + 'tiny-encoder.onnx', + 37_647_080, + '42c1d4cbf889632ba21ab6f0d4064c80209755f265ce5cd630db4a6793e7089c' + ], + [ + 'tiny-decoder.onnx', + 114_505_801, + 'e144c07dc6b55cece24392811f2d934b97013811f5e677d1315d341a0a74a25d' + ], + [ + 'tiny-tokens.txt', + 816_730, + 'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126' + ] + ] + ), + 'sense-voice-zh-en-ja-ko-yue': huggingFaceFiles( + 'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17', + '2365baeacb507f821a0c8120fcee3d484dba7a07', + [ + [ + 'model.int8.onnx', + 239_233_841, + 'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51' + ], + ['tokens.txt', 315_894, 'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc'] + ] + ) +} + +export type DownloadableSpeechModelId = keyof typeof MODEL_DOWNLOAD_FILES + +export function getSpeechModelDownloadMetadata(modelId: DownloadableSpeechModelId): { + downloadFiles: SpeechModelDownloadFile[] + files: string[] + sizeBytes: number +} { + const downloadFiles = MODEL_DOWNLOAD_FILES[modelId] + return { + downloadFiles, + files: downloadFiles.map(({ name }) => name), + sizeBytes: downloadFiles.reduce((total, { sizeBytes }) => total + sizeBytes, 0) + } +} diff --git a/src/main/speech/model-manager-download-resume.test.ts b/src/main/speech/model-manager-download-resume.test.ts index bbfc88e74..60645d1aa 100644 --- a/src/main/speech/model-manager-download-resume.test.ts +++ b/src/main/speech/model-manager-download-resume.test.ts @@ -19,9 +19,9 @@ vi.mock('electron', () => ({ })) type ModelManagerInternals = { - downloadArchiveWithRetry: ( + downloadFileWithRetry: ( url: string, - archivePath: string, + filePath: string, expectedSize: number, modelId: string, isAborted: () => boolean, @@ -29,7 +29,7 @@ type ModelManagerInternals = { ) => Promise downloadFile: ( url: string, - archivePath: string, + filePath: string, expectedSize: number, modelId: string, isAborted: () => boolean, @@ -141,11 +141,11 @@ describe('ModelManager download resume', () => { } }) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -154,13 +154,13 @@ describe('ModelManager download resume', () => { expect(netRequestMock).toHaveBeenCalledTimes(2) expect(second.sentHeaders.range).toBe('bytes=10-') - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } }) - it('uses a complete archive after a late transport failure without requesting past EOF', async () => { + it('uses a complete file after a late transport failure without requesting past EOF', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) try { scriptRequest(() => ({ @@ -170,11 +170,11 @@ describe('ModelManager download resume', () => { failWith: 'net::ERR_CONNECTION_RESET' })) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -182,13 +182,13 @@ describe('ModelManager download resume', () => { ) expect(netRequestMock).toHaveBeenCalledTimes(1) - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } }) - it('requests the remaining bytes when a clean range response ends before the archive total', async () => { + it('requests the remaining bytes when a clean range response ends before the file total', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) try { const first = scriptRequest((sentHeaders) => { @@ -214,12 +214,12 @@ describe('ModelManager download resume', () => { } }) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') - writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + const filePath = join(dir, 'model.bin') + writeFileSync(filePath, PAYLOAD.subarray(0, 10)) - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -229,7 +229,7 @@ describe('ModelManager download resume', () => { expect(netRequestMock).toHaveBeenCalledTimes(2) expect(first.sentHeaders.range).toBe('bytes=10-') expect(second.sentHeaders.range).toBe('bytes=15-') - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } @@ -263,12 +263,12 @@ describe('ModelManager download resume', () => { } }) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') - writeFileSync(archivePath, PAYLOAD.subarray(0, 1)) + const filePath = join(dir, 'model.bin') + writeFileSync(filePath, PAYLOAD.subarray(0, 1)) - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -276,7 +276,7 @@ describe('ModelManager download resume', () => { ) expect(netRequestMock).toHaveBeenCalledTimes(10) - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } @@ -286,20 +286,20 @@ describe('ModelManager download resume', () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) try { const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') let bytesWritten = 0 // Advances one byte per request against a total larger than the request // ceiling, so it makes forward progress forever without ever completing. const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => { bytesWritten += 1 - writeFileSync(archivePath, Buffer.alloc(bytesWritten)) + writeFileSync(filePath, Buffer.alloc(bytesWritten)) return Promise.resolve() }) await expect( - manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, 1_000_000, 'm', () => false, @@ -324,18 +324,18 @@ describe('ModelManager download resume', () => { // it takes (regression guard: a fixed failure budget used to abandon a // still-advancing large download around attempt 8). const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') const SLICE = 2 let delivered = 0 const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => { delivered = Math.min(delivered + SLICE, PAYLOAD.length) - writeFileSync(archivePath, PAYLOAD.subarray(0, delivered)) + writeFileSync(filePath, PAYLOAD.subarray(0, delivered)) return Promise.reject(new Error('net::ERR_CONNECTION_RESET')) }) - const download = manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + const download = manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -351,14 +351,14 @@ describe('ModelManager download resume', () => { await expect(outcome).resolves.toBe('resolved') expect(downloadFileMock).toHaveBeenCalledTimes(PAYLOAD.length / SLICE) - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { vi.useRealTimers() rmSync(dir, { recursive: true, force: true }) } }) - it('keeps the known archive total when Content-Range omits it', async () => { + it('keeps the known file total when Content-Range omits it', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) try { scriptRequest((sentHeaders) => { @@ -384,12 +384,12 @@ describe('ModelManager download resume', () => { } }) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') - writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + const filePath = join(dir, 'model.bin') + writeFileSync(filePath, PAYLOAD.subarray(0, 10)) - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -397,7 +397,7 @@ describe('ModelManager download resume', () => { ) expect(netRequestMock).toHaveBeenCalledTimes(2) - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } @@ -415,13 +415,13 @@ describe('ModelManager download resume', () => { chunks: [PAYLOAD.subarray(0, 10)] })) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') - writeFileSync(archivePath, PAYLOAD.subarray(0, 10)) + const filePath = join(dir, 'model.bin') + writeFileSync(filePath, PAYLOAD.subarray(0, 10)) const error = await manager .downloadFile( - 'https://example.com/model.tar.bz2', - archivePath, + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -435,7 +435,7 @@ describe('ModelManager download resume', () => { message: 'Invalid Content-Range for resume at byte 10', retryable: true }) - expect(existsSync(archivePath)).toBe(false) + expect(existsSync(filePath)).toBe(false) expect(mismatched.abortMock).toHaveBeenCalledTimes(1) } finally { rmSync(dir, { recursive: true, force: true }) @@ -457,11 +457,11 @@ describe('ModelManager download resume', () => { chunks: [PAYLOAD] })) const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') - await manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + await manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -469,7 +469,7 @@ describe('ModelManager download resume', () => { ) expect(netRequestMock).toHaveBeenCalledTimes(2) - expect(readFileSync(archivePath)).toEqual(PAYLOAD) + expect(readFileSync(filePath)).toEqual(PAYLOAD) } finally { rmSync(dir, { recursive: true, force: true }) } @@ -482,9 +482,9 @@ describe('ModelManager download resume', () => { const manager = new ModelManager(dir) as unknown as ModelManagerInternals await expect( - manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + manager.downloadFileWithRetry( + 'https://example.com/model.bin', + join(dir, 'model.bin'), PAYLOAD.length, 'm', () => false, @@ -509,8 +509,8 @@ describe('ModelManager download resume', () => { const error = await manager .downloadFile( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + 'https://example.com/model.bin', + join(dir, 'model.bin'), PAYLOAD.length, 'm', () => false @@ -529,7 +529,7 @@ describe('ModelManager download resume', () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-')) try { const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const archivePath = join(dir, 'model.tar.bz2') + const filePath = join(dir, 'model.bin') const rateLimitError = Object.assign(new Error('HTTP 429'), { httpStatusCode: 429, retryAfterMs: 3_000 @@ -538,12 +538,12 @@ describe('ModelManager download resume', () => { .spyOn(manager, 'downloadFile') .mockRejectedValueOnce(rateLimitError) .mockImplementationOnce(() => { - writeFileSync(archivePath, PAYLOAD) + writeFileSync(filePath, PAYLOAD) return Promise.resolve() }) - const download = manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - archivePath, + const download = manager.downloadFileWithRetry( + 'https://example.com/model.bin', + filePath, PAYLOAD.length, 'm', () => false, @@ -572,9 +572,9 @@ describe('ModelManager download resume', () => { const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockRejectedValue(rateLimitError) await expect( - manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + manager.downloadFileWithRetry( + 'https://example.com/model.bin', + join(dir, 'model.bin'), PAYLOAD.length, 'm', () => false, @@ -597,9 +597,9 @@ describe('ModelManager download resume', () => { .spyOn(manager, 'downloadFile') .mockRejectedValue(new Error('net::ERR_CONNECTION_RESET')) - const download = manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + const download = manager.downloadFileWithRetry( + 'https://example.com/model.bin', + join(dir, 'model.bin'), PAYLOAD.length, 'm', () => false, @@ -645,9 +645,9 @@ describe('ModelManager download resume', () => { const controller = new AbortController() const manager = new ModelManager(dir) as unknown as ModelManagerInternals - const download = manager.downloadArchiveWithRetry( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + const download = manager.downloadFileWithRetry( + 'https://example.com/model.bin', + join(dir, 'model.bin'), PAYLOAD.length, 'm', () => false, diff --git a/src/main/speech/model-manager-stream-cleanup.test.ts b/src/main/speech/model-manager-stream-cleanup.test.ts index fecec7980..dcebaca37 100644 --- a/src/main/speech/model-manager-stream-cleanup.test.ts +++ b/src/main/speech/model-manager-stream-cleanup.test.ts @@ -64,8 +64,8 @@ describe('ModelManager stream cleanup', () => { const manager = new ModelManager(dir) as unknown as ModelManagerInternals const download = manager.downloadFile( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + 'https://example.com/model.bin', + join(dir, 'model.bin'), 4, 'm', () => false diff --git a/src/main/speech/model-manager-windows-path.test.ts b/src/main/speech/model-manager-windows-path.test.ts index b383b16bc..3b72cc832 100644 --- a/src/main/speech/model-manager-windows-path.test.ts +++ b/src/main/speech/model-manager-windows-path.test.ts @@ -1,4 +1,4 @@ -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' @@ -84,13 +84,16 @@ describe('ModelManager Windows model path handling', () => { appGetPathMock.mockImplementation((name: string) => name === 'userData' ? userDataDir : join(dir, name) ) - const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local') + const manifest = SPEECH_MODEL_CATALOG.find( + (model) => model.id === 'zipformer-streaming-zh-14m' + ) expect(manifest?.files).toBeDefined() const legacyModelDir = join(userDataDir, 'speech-models', manifest!.id) - for (const file of manifest!.files ?? []) { - const filePath = join(legacyModelDir, file) + for (const file of manifest!.downloadFiles ?? []) { + const filePath = join(legacyModelDir, file.name) mkdirSync(dirname(filePath), { recursive: true }) - writeFileSync(filePath, 'model file') + writeFileSync(filePath, '') + truncateSync(filePath, file.sizeBytes) } const manager = new ModelManager() @@ -120,12 +123,15 @@ describe('ModelManager Windows model path handling', () => { appGetPathMock.mockImplementation((name: string) => name === 'userData' ? userDataDir : join(dir, name) ) - const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local') + const manifest = SPEECH_MODEL_CATALOG.find( + (model) => model.id === 'zipformer-streaming-zh-14m' + ) const legacyModelDir = join(userDataDir, 'speech-models', manifest!.id) - for (const file of manifest!.files ?? []) { - const filePath = join(legacyModelDir, file) + for (const file of manifest!.downloadFiles ?? []) { + const filePath = join(legacyModelDir, file.name) mkdirSync(dirname(filePath), { recursive: true }) - writeFileSync(filePath, 'model file') + writeFileSync(filePath, '') + truncateSync(filePath, file.sizeBytes) } const manager = new ModelManager() diff --git a/src/main/speech/model-manager.test.ts b/src/main/speech/model-manager.test.ts index 07d78749b..006ba707f 100644 --- a/src/main/speech/model-manager.test.ts +++ b/src/main/speech/model-manager.test.ts @@ -1,15 +1,14 @@ import { createHash } from 'node:crypto' -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { existsSync, mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { beforeEach, describe, expect, it, vi } from 'vitest' import { SPEECH_MODEL_CATALOG } from './model-catalog' import { ModelManager } from './model-manager' -const { hasOpenAiSpeechApiKeyMock, netRequestMock, spawnMock } = vi.hoisted(() => ({ +const { hasOpenAiSpeechApiKeyMock, netRequestMock } = vi.hoisted(() => ({ hasOpenAiSpeechApiKeyMock: vi.fn(), - netRequestMock: vi.fn(), - spawnMock: vi.fn() + netRequestMock: vi.fn() })) vi.mock('electron', () => ({ @@ -21,17 +20,22 @@ vi.mock('electron', () => ({ } })) -vi.mock('child_process', async () => { - const actual = await vi.importActual('child_process') - return { ...(actual as Record), spawn: spawnMock } -}) - vi.mock('./openai-api-key-store', () => ({ hasOpenAiSpeechApiKey: hasOpenAiSpeechApiKeyMock })) type ModelManagerInternals = { - verifyArchiveSha256: (archivePath: string, expectedSha256: string) => Promise + verifyFileSha256: (filePath: string, expectedSha256: string) => Promise + downloadFileWithRetry: ( + url: string, + filePath: string, + expectedSize: number, + modelId: string, + isAborted: () => boolean, + signal: AbortSignal, + completedBytes?: number, + modelTotalBytes?: number + ) => Promise downloadFile: ( url: string, dest: string, @@ -40,12 +44,6 @@ type ModelManagerInternals = { isAborted: () => boolean, signal?: AbortSignal ) => Promise - extractArchive: ( - archivePath: string, - destDir: string, - modelId: string, - isAborted: () => boolean - ) => Promise } describe('ModelManager', () => { @@ -53,28 +51,38 @@ describe('ModelManager', () => { netRequestMock.mockReset() hasOpenAiSpeechApiKeyMock.mockReset() hasOpenAiSpeechApiKeyMock.mockReturnValue(false) - spawnMock.mockReset() }) - it('requires pinned SHA-256 hashes for every catalog archive', () => { + it('requires pinned, internally consistent metadata for every model file', () => { for (const manifest of SPEECH_MODEL_CATALOG) { if (manifest.provider !== 'local') { continue } - expect(manifest.archiveSha256).toMatch(/^[a-f0-9]{64}$/) + expect(manifest.downloadFiles?.length).toBeGreaterThan(0) + expect(manifest.files).toEqual(manifest.downloadFiles?.map(({ name }) => name)) + expect(manifest.sizeBytes).toBe( + manifest.downloadFiles?.reduce((total, { sizeBytes }) => total + sizeBytes, 0) + ) + for (const file of manifest.downloadFiles ?? []) { + expect(file.url).toMatch( + /^https:\/\/huggingface\.co\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\// + ) + expect(file.sha256).toMatch(/^[a-f0-9]{64}$/) + expect(file.sizeBytes).toBeGreaterThan(0) + } } }) - it('verifies downloaded archive hashes before extraction', async () => { + it('verifies downloaded model file hashes before installation', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) try { - const archivePath = join(dir, 'model.tar.bz2') - writeFileSync(archivePath, 'known archive bytes') - const expected = createHash('sha256').update('known archive bytes').digest('hex') + const filePath = join(dir, 'model.onnx') + writeFileSync(filePath, 'known model bytes') + const expected = createHash('sha256').update('known model bytes').digest('hex') const manager = new ModelManager(dir) as unknown as ModelManagerInternals - await expect(manager.verifyArchiveSha256(archivePath, expected)).resolves.toBeUndefined() - await expect(manager.verifyArchiveSha256(archivePath, '0'.repeat(64))).rejects.toThrow( + await expect(manager.verifyFileSha256(filePath, expected)).resolves.toBeUndefined() + await expect(manager.verifyFileSha256(filePath, '0'.repeat(64))).rejects.toThrow( /integrity verification/ ) } finally { @@ -89,8 +97,8 @@ describe('ModelManager', () => { await expect( manager.downloadFile( - 'http://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + 'http://example.com/model.bin', + join(dir, 'model.bin'), 1, 'm', () => false @@ -101,6 +109,46 @@ describe('ModelManager', () => { } }) + it('installs individually verified model files through a staging directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) + try { + const manifest = SPEECH_MODEL_CATALOG.find( + (model) => model.id === 'zipformer-streaming-zh-14m' + )! + const manager = new ModelManager(dir) + const internals = manager as unknown as ModelManagerInternals + const downloadMock = vi + .spyOn(internals, 'downloadFileWithRetry') + .mockImplementation(async (_url, filePath, expectedSize) => { + writeFileSync(filePath, '') + truncateSync(filePath, expectedSize) + }) + const verifyMock = vi.spyOn(internals, 'verifyFileSha256').mockResolvedValue() + + await manager.downloadModel(manifest.id) + + const modelDir = manager.getModelDir(manifest.id) + expect(downloadMock).toHaveBeenCalledTimes(manifest.downloadFiles?.length ?? 0) + expect(verifyMock).toHaveBeenCalledTimes(manifest.downloadFiles?.length ?? 0) + let expectedOffset = 0 + for (const [index, file] of (manifest.downloadFiles ?? []).entries()) { + expect(downloadMock.mock.calls[index]?.slice(6)).toEqual([ + expectedOffset, + manifest.sizeBytes + ]) + expectedOffset += file.sizeBytes + expect(existsSync(join(modelDir, file.name))).toBe(true) + } + expect(existsSync(`${modelDir}.partial`)).toBe(false) + await expect(manager.getModelState(manifest.id)).resolves.toEqual({ + id: manifest.id, + status: 'ready' + }) + } finally { + rmSync(dir, { recursive: true, force: true }) + } + }) + it('marks OpenAI transcription models ready only when an API key is configured', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) try { @@ -125,14 +173,17 @@ describe('ModelManager', () => { it('deletes a ready local model and reports it as not downloaded', async () => { const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) try { - const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local') + const manifest = SPEECH_MODEL_CATALOG.find( + (model) => model.id === 'zipformer-streaming-zh-14m' + ) expect(manifest?.files).toBeDefined() const manager = new ModelManager(dir) const modelDir = manager.getModelDir(manifest!.id) - for (const file of manifest!.files ?? []) { - const path = join(modelDir, file) + for (const file of manifest!.downloadFiles ?? []) { + const path = join(modelDir, file.name) mkdirSync(dirname(path), { recursive: true }) - writeFileSync(path, 'model file') + writeFileSync(path, '') + truncateSync(path, file.sizeBytes) } await expect(manager.getModelState(manifest!.id)).resolves.toEqual({ @@ -288,8 +339,8 @@ describe('ModelManager', () => { const manager = new ModelManager(dir) as unknown as ModelManagerInternals const download = manager.downloadFile( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + 'https://example.com/model.bin', + join(dir, 'model.bin'), 1, 'm', () => true, @@ -370,8 +421,8 @@ describe('ModelManager', () => { const manager = new ModelManager(dir) as unknown as ModelManagerInternals const download = manager.downloadFile( - 'https://example.com/model.tar.bz2', - join(dir, 'model.tar.bz2'), + 'https://example.com/model.bin', + join(dir, 'model.bin'), 1, 'm', () => false @@ -397,59 +448,4 @@ describe('ModelManager', () => { rmSync(dir, { recursive: true, force: true }) } }) - - it('clears extraction abort polling when the child does not close', async () => { - vi.useFakeTimers() - const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-')) - try { - const handlers: Record void)[]> = { - close: [], - error: [] - } - const stderrHandlers: ((chunk: Buffer) => void)[] = [] - const child = { - stderr: { - on: vi.fn((_event: string, cb: (chunk: Buffer) => void) => { - stderrHandlers.push(cb) - return child.stderr - }), - off: vi.fn((_event: string, cb: (chunk: Buffer) => void) => { - const index = stderrHandlers.indexOf(cb) - if (index !== -1) { - stderrHandlers.splice(index, 1) - } - return child.stderr - }) - }, - kill: vi.fn(), - on: vi.fn((event: string, cb: (arg?: unknown) => void) => { - handlers[event]?.push(cb) - return child - }), - off: vi.fn((event: string, cb: (arg?: unknown) => void) => { - handlers[event] = handlers[event]?.filter((handler) => handler !== cb) ?? [] - return child - }) - } - spawnMock.mockReturnValue(child) - const manager = new ModelManager(dir) as unknown as ModelManagerInternals - - const extraction = manager.extractArchive(join(dir, 'model.tar.bz2'), dir, 'm', () => true) - const rejection = expect(extraction).rejects.toThrow('Aborted') - await vi.advanceTimersByTimeAsync(250) - await rejection - - expect(child.kill).toHaveBeenCalledWith('SIGKILL') - expect(child.kill).toHaveBeenCalledTimes(1) - expect(handlers.close).toHaveLength(0) - expect(handlers.error).toHaveLength(0) - expect(stderrHandlers).toHaveLength(0) - - vi.advanceTimersByTime(1000) - expect(child.kill).toHaveBeenCalledTimes(1) - } finally { - vi.useRealTimers() - rmSync(dir, { recursive: true, force: true }) - } - }) }) diff --git a/src/main/speech/model-manager.ts b/src/main/speech/model-manager.ts index 2b8bca8bb..9125ffa49 100644 --- a/src/main/speech/model-manager.ts +++ b/src/main/speech/model-manager.ts @@ -1,4 +1,4 @@ -/* eslint-disable max-lines -- Why: model download, checksum, extraction, and cleanup share one state machine so progress/error transitions stay coupled. */ +/* eslint-disable max-lines -- Why: model download, checksum, retry, and cleanup share one state machine so progress/error transitions stay coupled. */ import { app, net } from 'electron' import { join, resolve, relative } from 'node:path' import { @@ -9,10 +9,9 @@ import { rmSync, statSync } from 'node:fs' -import { readdir, rm } from 'node:fs/promises' +import { rename, rm } from 'node:fs/promises' import { createHash } from 'node:crypto' import { pipeline } from 'node:stream/promises' -import { spawn } from 'node:child_process' import type { SpeechModelManifest, SpeechModelState, @@ -20,7 +19,6 @@ import type { } from '../../shared/speech-types' import { SPEECH_MODEL_CATALOG, getCatalogModel, isLocalSpeechModel } from './model-catalog' import { hasOpenAiSpeechApiKey } from './openai-api-key-store' -import { resolveTarExecutable } from './tar-executable' import { getSpeechModelCacheDirCandidates, migrateSpeechModelCacheIfNeeded, @@ -42,7 +40,11 @@ type HttpStatusError = Error & { retryAfterMs?: number retryable?: boolean } -type DownloadTotals = { totalBytes: number } +type DownloadTotals = { + totalBytes: number + completedBytes: number + modelTotalBytes: number +} type ContentRange = { start: number; end: number; totalBytes?: number } const DOWNLOAD_IDLE_TIMEOUT_MS = 120_000 @@ -254,10 +256,16 @@ export class ModelManager { } private validateModelFiles(manifest: SpeechModelManifest, modelDir: string): boolean { - if (!manifest.files) { + if (!manifest.downloadFiles) { return false } - return manifest.files.every((f) => existsSync(join(modelDir, f))) + return manifest.downloadFiles.every(({ name, sizeBytes }) => { + try { + return statSync(join(modelDir, name)).size === sizeBytes + } catch { + return false + } + }) } async downloadModel(modelId: string): Promise { @@ -273,7 +281,7 @@ export class ModelManager { if (!isLocalSpeechModel(manifest)) { throw new Error(`Model does not support downloads: ${modelId}`) } - if (!manifest.downloadUrl || !manifest.archiveSha256 || !manifest.sizeBytes) { + if (!manifest.downloadFiles?.length || !manifest.sizeBytes) { throw new Error(`Model download metadata missing: ${modelId}`) } @@ -285,15 +293,16 @@ export class ModelManager { this.updateState(modelId, 'downloading', 0) - const archivePath = join(this.modelsDir, `${modelId}.tar.bz2`) - // Why: resume appends, so a leftover archive from a crashed run would corrupt the download. + const stagingDir = `${modelDir}.partial` + const legacyArchivePath = join(this.modelsDir, `${modelId}.tar.bz2`) + // Why: resuming an unverified file left by a crashed process could preserve corrupt bytes. + rmSync(stagingDir, { recursive: true, force: true }) try { - if (existsSync(archivePath)) { - rmSync(archivePath) - } + rmSync(legacyArchivePath, { force: true }) } catch { - // best-effort; the first (non-resumed) attempt truncates on write + // best-effort legacy cleanup } + mkdirSync(stagingDir, { recursive: true }) let aborted = false const abortController = new AbortController() @@ -307,69 +316,34 @@ export class ModelManager { this.activeDownloads.set(modelId, handle) try { - await this.downloadArchiveWithRetry( - manifest.downloadUrl, - archivePath, - manifest.sizeBytes, + await this.downloadModelFiles( + manifest, + stagingDir, modelId, () => aborted, abortController.signal ) if (aborted) { - this.cleanup(modelId, archivePath) return } - await this.verifyArchiveSha256(archivePath, manifest.archiveSha256) - - if (aborted) { - this.cleanup(modelId, archivePath) - return - } - - this.updateState(modelId, 'extracting') - await this.extractArchive(archivePath, this.modelsDir, modelId, () => aborted) - - if (aborted) { - this.cleanup(modelId, archivePath) - return - } - - if (!this.validateModelFiles(manifest, modelDir)) { - // Why: some archives nest files in a subdir; scan one level down and move them up. - await this.flattenNestedDir(modelDir, manifest) - } - - if (aborted) { - this.cleanup(modelId, archivePath) - return - } - - if (!this.validateModelFiles(manifest, modelDir)) { - throw new Error('Model files missing after extraction') - } - + await rm(modelDir, { recursive: true, force: true }) + await rename(stagingDir, modelDir) this.updateState(modelId, 'ready') } catch (err) { if (!aborted) { console.error('[speech] Model download failed:', modelId, err) this.updateState(modelId, 'error', undefined, String(err)) } - this.cleanup(modelId, archivePath) + this.removeModelDownloadFiles(modelDir, stagingDir, legacyArchivePath) if (!aborted) { // Why: the settings UI awaits this to surface failures; stay quiet on cancellation, rethrow real errors. throw err } } finally { this.activeDownloads.delete(modelId) - try { - if (existsSync(archivePath)) { - rmSync(archivePath) - } - } catch { - // best-effort archive cleanup - } + this.removeModelDownloadStaging(stagingDir, legacyArchivePath) } } @@ -395,6 +369,8 @@ export class ModelManager { if (existsSync(modelDir)) { await rm(modelDir, { recursive: true, force: true }) } + await rm(`${modelDir}.partial`, { recursive: true, force: true }) + await rm(join(this.modelsDir, `${modelId}.tar.bz2`), { force: true }) // Why: also delete the pre-migration copy, or the next launch re-migrates it and resurrects the model. if (this.migrationSourceDir) { const sourceModelDir = this.getSafeModelDir(modelId, this.migrationSourceDir) @@ -420,28 +396,75 @@ export class ModelManager { } } - private getPartialArchiveBytes(archivePath: string): number { + private async downloadModelFiles( + manifest: SpeechModelManifest, + stagingDir: string, + modelId: string, + isAborted: () => boolean, + signal: AbortSignal + ): Promise { + if (!manifest.downloadFiles?.length || !manifest.sizeBytes) { + throw new Error(`Model download metadata missing: ${modelId}`) + } + + let completedBytes = 0 + for (const file of manifest.downloadFiles) { + if ( + !file.name || + file.name === '.' || + file.name === '..' || + file.name.includes('/') || + file.name.includes('\\') + ) { + throw new Error(`Invalid model download filename: ${file.name}`) + } + const filePath = join(stagingDir, file.name) + await this.downloadFileWithRetry( + file.url, + filePath, + file.sizeBytes, + modelId, + isAborted, + signal, + completedBytes, + manifest.sizeBytes + ) + if (isAborted()) { + return + } + await this.verifyFileSha256(filePath, file.sha256) + completedBytes += file.sizeBytes + } + + if (!this.validateModelFiles(manifest, stagingDir)) { + throw new Error('Model files missing after download') + } + } + + private getPartialDownloadBytes(filePath: string): number { try { - return statSync(archivePath).size + return statSync(filePath).size } catch { return 0 } } - private async downloadArchiveWithRetry( + private async downloadFileWithRetry( url: string, - archivePath: string, + filePath: string, expectedSize: number, modelId: string, isAborted: () => boolean, - signal: AbortSignal + signal: AbortSignal, + completedBytes = 0, + modelTotalBytes = expectedSize ): Promise { let requestCount = 0 let noProgressStreak = 0 - const totals: DownloadTotals = { totalBytes: expectedSize } + const totals: DownloadTotals = { totalBytes: expectedSize, completedBytes, modelTotalBytes } for (;;) { requestCount += 1 - const offset = this.getPartialArchiveBytes(archivePath) + const offset = this.getPartialDownloadBytes(filePath) // Why: transport can fail after the last byte hits disk; the SHA-256 check is the real completion test. if (offset === totals.totalBytes) { return @@ -459,7 +482,7 @@ export class ModelManager { // Why: restart from the canonical URL, not the last redirect, because signed CDN redirect URLs expire. await this.downloadFile( url, - archivePath, + filePath, expectedSize, modelId, isAborted, @@ -468,7 +491,7 @@ export class ModelManager { offset, totals ) - const receivedBytes = this.getPartialArchiveBytes(archivePath) + const receivedBytes = this.getPartialDownloadBytes(filePath) if (receivedBytes === totals.totalBytes) { return } @@ -492,7 +515,7 @@ export class ModelManager { if (isAborted() || signal.aborted) { throw err } - const receivedBytes = this.getPartialArchiveBytes(archivePath) + const receivedBytes = this.getPartialDownloadBytes(filePath) if (receivedBytes === totals.totalBytes) { return } @@ -666,7 +689,7 @@ export class ModelManager { (parsedLength <= 0 || parsedLength === contentRange.end - contentRange.start + 1) if (resumeOffset > 0 && response.statusCode === 206 && !resumed) { - // Why: appending an unverified range can silently corrupt the archive; discard and retry from byte zero. + // Why: appending an unverified range can silently corrupt the file; discard and retry from byte zero. try { rmSync(dest) } catch { @@ -728,7 +751,11 @@ export class ModelManager { return } downloaded += chunk.length - const progress = Math.min(0.9, (progressBase + downloaded) / totalSize) + const progress = Math.min( + 0.9, + ((totals?.completedBytes ?? 0) + progressBase + downloaded) / + (totals?.modelTotalBytes ?? totalSize) + ) this.updateState(modelId, 'downloading', progress) } @@ -765,10 +792,10 @@ export class ModelManager { }) } - private verifyArchiveSha256(archivePath: string, expectedSha256: string): Promise { + private verifyFileSha256(filePath: string, expectedSha256: string): Promise { return new Promise((resolve, reject) => { const hash = createHash('sha256') - const stream = createReadStream(archivePath) + const stream = createReadStream(filePath) let settled = false const cleanup = (): void => { @@ -801,8 +828,8 @@ export class ModelManager { const onEnd = (): void => { const actualSha256 = hash.digest('hex') if (actualSha256 !== expectedSha256.toLowerCase()) { - // Why: archives feed native parsers, so verify contents against compromised/redirected release assets. - settleReject(new Error('Downloaded model archive failed integrity verification')) + // Why: model artifacts feed native runtimes, so verify every downloaded file before installation. + settleReject(new Error('Downloaded model file failed integrity verification')) return } settleResolve() @@ -814,125 +841,24 @@ export class ModelManager { }) } - private extractArchive( - archivePath: string, - destDir: string, - modelId: string, - isAborted: () => boolean - ): Promise { - const modelDir = join(destDir, modelId) - mkdirSync(modelDir, { recursive: true }) - - return new Promise((resolve, reject) => { - // Why: spawn (not exec) so slow bzip2 stderr can't overflow exec's 1MB maxBuffer and silently kill the process. - const tarExecutable = resolveTarExecutable() - const child = spawn( - tarExecutable, - ['-xjf', archivePath, '-C', modelDir, '--strip-components=1'], - { - stdio: ['ignore', 'ignore', 'pipe'], - windowsHide: true - } - ) - - let stderr = '' - let settled = false - let timeout: ReturnType | null = null - let abortPoll: ReturnType | null = null - const cleanup = (): void => { - if (timeout) { - clearTimeout(timeout) - timeout = null - } - if (abortPoll) { - clearInterval(abortPoll) - abortPoll = null - } - child.stderr?.off('data', onStderrData) - child.off('close', onClose) - child.off('error', onError) - } - const fail = (error: Error, killChild = false): void => { - if (settled) { - return - } - settled = true - cleanup() - if (killChild) { - child.kill('SIGKILL') - } - reject(error) - } - const onStderrData = (chunk: Buffer): void => { - stderr += chunk.toString() - } - const onClose = (code: number | null): void => { - if (settled) { - return - } - settled = true - cleanup() - if (code === 0) { - resolve() - } else { - reject(new Error(`tar exited with code ${code}: ${stderr.slice(0, 500)}`)) - } - } - const onError = (err: Error): void => { - fail(err) - } - - child.stderr?.on('data', onStderrData) - timeout = setTimeout(() => { - fail(new Error('Extraction timed out after 10 minutes'), true) - }, 600_000) - abortPoll = setInterval(() => { - if (isAborted()) { - // Why: a wedged child may never emit close/error, so abort must kill it here. - fail(new Error('Aborted'), true) - } - }, 250) - - child.on('close', onClose) - child.on('error', onError) - }) - } - - private async flattenNestedDir(modelDir: string, manifest: SpeechModelManifest): Promise { - if (!manifest.files) { - return - } - const entries = await readdir(modelDir, { withFileTypes: true }) - for (const entry of entries) { - if (entry.isDirectory()) { - const nestedDir = join(modelDir, entry.name) - const nestedFiles = await readdir(nestedDir) - const hasExpected = manifest.files.some((f) => nestedFiles.includes(f)) - if (hasExpected) { - const { rename: fsRename } = await import('node:fs/promises') - for (const file of nestedFiles) { - await fsRename(join(nestedDir, file), join(modelDir, file)) - } - await rm(nestedDir, { recursive: true, force: true }) - return - } + private removeModelDownloadStaging(stagingDir: string, legacyArchivePath: string): void { + for (const path of [stagingDir, legacyArchivePath]) { + try { + rmSync(path, { recursive: true, force: true }) + } catch { + // best-effort } } } - private cleanup(modelId: string, archivePath: string): void { + private removeModelDownloadFiles( + modelDir: string, + stagingDir: string, + legacyArchivePath: string + ): void { + this.removeModelDownloadStaging(stagingDir, legacyArchivePath) try { - if (existsSync(archivePath)) { - rmSync(archivePath) - } - } catch { - // best-effort - } - const modelDir = this.getModelDir(modelId) - try { - if (existsSync(modelDir)) { - rmSync(modelDir, { recursive: true }) - } + rmSync(modelDir, { recursive: true, force: true }) } catch { // best-effort } diff --git a/src/main/speech/stt-worker-model-config.ts b/src/main/speech/stt-worker-model-config.ts index de4687d9f..ff9382fe3 100644 --- a/src/main/speech/stt-worker-model-config.ts +++ b/src/main/speech/stt-worker-model-config.ts @@ -26,9 +26,7 @@ export function resolveTokens(files: string[], modelDir: string): string { return join(modelDir, match) } -// Why: BPE models need a vocab file for hotwords token matching. The file -// ships in the model archive but isn't listed in the manifest. We discover -// it at runtime to avoid breaking existing downloads. +// Why: BPE models need a vocab file for hotwords token matching, but older caches may omit it. function discoverBpeVocab(modelDir: string): string | undefined { try { const entries = readdirSync(modelDir) diff --git a/src/main/speech/tar-executable.ts b/src/main/speech/tar-executable.ts deleted file mode 100644 index d097eb1b8..000000000 --- a/src/main/speech/tar-executable.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { existsSync } from 'node:fs' -import { win32 as pathWin32 } from 'node:path' - -export function resolveTarExecutable( - options: { - platform?: NodeJS.Platform - env?: NodeJS.ProcessEnv - exists?: (path: string) => boolean - } = {} -): string { - const platform = options.platform ?? process.platform - if (platform !== 'win32') { - return 'tar' - } - - const env = options.env ?? process.env - const systemRoot = env.SystemRoot ?? env.WINDIR ?? 'C:\\Windows' - const candidate = pathWin32.join(systemRoot, 'System32', 'tar.exe') - const exists = options.exists ?? existsSync - if (exists(candidate)) { - return candidate - } - - // Why: packaged Windows apps can have a stripped PATH. Use the OS tar - // location explicitly, and fail with a repairable error if it is absent. - throw new Error(`Windows tar.exe not found at ${candidate}`) -} diff --git a/src/shared/speech-types.ts b/src/shared/speech-types.ts index 2d89bbb49..93e1989db 100644 --- a/src/shared/speech-types.ts +++ b/src/shared/speech-types.ts @@ -9,6 +9,13 @@ export type SpeechModelProvider = 'local' | 'openai' export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe' +export type SpeechModelDownloadFile = { + name: string + url: string + sizeBytes: number + sha256: string +} + export type SpeechModelManifest = { id: string label: string @@ -17,9 +24,7 @@ export type SpeechModelManifest = { provider: SpeechModelProvider language: string sizeBytes?: number - downloadUrl?: string - archiveSha256?: string - archiveFormat?: 'tar.bz2' + downloadFiles?: SpeechModelDownloadFile[] files?: string[] sampleRate: number streaming: boolean