fix(speech): download verified model artifacts directly (#10735)
This commit is contained in:
Neil
GitHub
parent
ab1c37889a
commit
c30a0ea685
|
|
@ -12,14 +12,11 @@ describe('SPEECH_MODEL_CATALOG', () => {
|
|||
expect(manifest?.streaming).toBe(false)
|
||||
expect(manifest?.sampleRate).toBe(16000)
|
||||
expect(manifest?.files).toEqual(['model.int8.onnx', 'tokens.txt'])
|
||||
expect(manifest?.archiveFormat).toBe('tar.bz2')
|
||||
expect(manifest?.sizeBytes).toBe(489_389_564)
|
||||
expect(manifest?.downloadUrl).toBe(
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2'
|
||||
)
|
||||
expect(manifest?.archiveSha256).toBe(
|
||||
'4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b'
|
||||
)
|
||||
expect(manifest?.sizeBytes).toBe(655_571_161)
|
||||
expect(manifest?.downloadFiles?.map(({ name }) => name)).toEqual([
|
||||
'model.int8.onnx',
|
||||
'tokens.txt'
|
||||
])
|
||||
})
|
||||
|
||||
it('has unique ids across the catalog', () => {
|
||||
|
|
@ -42,14 +39,10 @@ describe('SPEECH_MODEL_CATALOG', () => {
|
|||
expect(model?.files).toEqual(['model.int8.onnx', 'tokens.txt'])
|
||||
})
|
||||
|
||||
it('downloads the upstream int8-only SenseVoice archive', () => {
|
||||
it('downloads only the pinned SenseVoice runtime files', () => {
|
||||
const model = getCatalogModel('sense-voice-zh-en-ja-ko-yue')
|
||||
expect(model?.sizeBytes).toBe(163_002_883)
|
||||
expect(model?.downloadUrl).toBe(
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2'
|
||||
)
|
||||
expect(model?.archiveSha256).toBe(
|
||||
'7d1efa2138a65b0b488df37f8b89e3d91a60676e416f515b952358d83dfd347e'
|
||||
)
|
||||
expect(model?.sizeBytes).toBe(239_549_735)
|
||||
expect(model?.downloadFiles).toHaveLength(2)
|
||||
expect(model?.downloadFiles?.map(({ name }) => name)).toEqual(['model.int8.onnx', 'tokens.txt'])
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import type { SpeechModelManifest } from '../../shared/speech-types'
|
||||
import { getSpeechModelDownloadMetadata } from './model-download-catalog'
|
||||
|
||||
// Why: sizeBytes must be the exact upstream asset size — it is the UI size
|
||||
// label and the download-progress denominator when content-length is missing.
|
||||
export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
||||
{
|
||||
id: 'parakeet-tdt-0.6b-v3-int8',
|
||||
|
|
@ -11,12 +10,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'multilingual',
|
||||
sizeBytes: 487_170_055,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8.tar.bz2',
|
||||
archiveSha256: '5793d0fd397c5778d2cf2126994d58e9d56b1be7c04d13c7a15bb1b4eafb16bf',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('parakeet-tdt-0.6b-v3-int8'),
|
||||
sampleRate: 16000,
|
||||
streaming: false,
|
||||
modelingUnit: 'bpe',
|
||||
|
|
@ -30,12 +24,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'en',
|
||||
sizeBytes: 482_468_385,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8.tar.bz2',
|
||||
archiveSha256: '157c157bc51155e03e37d2466522a3a737dd9c72bb25f36eb18912964161e1ad',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('parakeet-tdt-0.6b-v2-int8'),
|
||||
sampleRate: 16000,
|
||||
streaming: false,
|
||||
modelingUnit: 'bpe'
|
||||
|
|
@ -47,17 +36,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'zh-en',
|
||||
sizeBytes: 511_274_346,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20.tar.bz2',
|
||||
archiveSha256: '27ffbd9ee24ad186d99acc2f6354d7992b27bcab490812510665fa8f9389c5f8',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: [
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
'tokens.txt'
|
||||
],
|
||||
...getSpeechModelDownloadMetadata('zipformer-bilingual-zh-en'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'cjkchar+bpe'
|
||||
|
|
@ -70,12 +49,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'paraformer',
|
||||
provider: 'local',
|
||||
language: 'zh-en',
|
||||
sizeBytes: 1_047_319_737,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2',
|
||||
archiveSha256: '5462a1fce42693deae572af1e8c4687124b12aa85fe61ff4d3168bb5280e205f',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['encoder.int8.onnx', 'decoder.int8.onnx', 'tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('paraformer-bilingual-zh-en'),
|
||||
sampleRate: 16000,
|
||||
streaming: true
|
||||
},
|
||||
|
|
@ -86,17 +60,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'en',
|
||||
sizeBytes: 127_887_156,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17.tar.bz2',
|
||||
archiveSha256: '9c559283e8498d3fe95913c79ca1cb454bb26281ac2b102b41306c7d752765d9',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: [
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
'tokens.txt'
|
||||
],
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-en-20m'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'bpe'
|
||||
|
|
@ -108,17 +72,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'zh',
|
||||
sizeBytes: 74_004_050,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23.tar.bz2',
|
||||
archiveSha256: '2cbd71b640d9c37d3784f29367333a4577b0398b62e9deeed418170b081cba8b',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: [
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
'tokens.txt'
|
||||
],
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-zh-14m'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'cjkchar'
|
||||
|
|
@ -130,17 +84,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'transducer',
|
||||
provider: 'local',
|
||||
language: 'ko',
|
||||
sizeBytes: 418_218_652,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-zipformer-korean-2024-06-16.tar.bz2',
|
||||
archiveSha256: 'e346a5882a409650472be17326237e24df7bf409db6b4a8a52e1a61422bf2500',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: [
|
||||
'encoder-epoch-99-avg-1.int8.onnx',
|
||||
'decoder-epoch-99-avg-1.int8.onnx',
|
||||
'joiner-epoch-99-avg-1.int8.onnx',
|
||||
'tokens.txt'
|
||||
],
|
||||
...getSpeechModelDownloadMetadata('zipformer-streaming-korean'),
|
||||
sampleRate: 16000,
|
||||
streaming: true,
|
||||
modelingUnit: 'bpe'
|
||||
|
|
@ -152,12 +96,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'nemo-ctc',
|
||||
provider: 'local',
|
||||
language: 'ja',
|
||||
sizeBytes: 489_389_564,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2',
|
||||
archiveSha256: '4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['model.int8.onnx', 'tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('parakeet-tdt-ctc-0.6b-ja-int8'),
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
|
|
@ -168,12 +107,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'whisper',
|
||||
provider: 'local',
|
||||
language: 'multilingual',
|
||||
sizeBytes: 116_204_861,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-whisper-tiny.tar.bz2',
|
||||
archiveSha256: 'c46116994e539aa165266d96b325252728429c12535eb9d8b6a2b10f129e66b1',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['tiny-encoder.onnx', 'tiny-decoder.onnx', 'tiny-tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('whisper-tiny'),
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
|
|
@ -185,12 +119,7 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
type: 'senseVoice',
|
||||
provider: 'local',
|
||||
language: 'multilingual',
|
||||
sizeBytes: 163_002_883,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2',
|
||||
archiveSha256: '7d1efa2138a65b0b488df37f8b89e3d91a60676e416f515b952358d83dfd347e',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['model.int8.onnx', 'tokens.txt'],
|
||||
...getSpeechModelDownloadMetadata('sense-voice-zh-en-ja-ko-yue'),
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,230 @@
|
|||
import type { SpeechModelDownloadFile } from '../../shared/speech-types'
|
||||
|
||||
type DownloadFileSpec = readonly [name: string, sizeBytes: number, sha256: string]
|
||||
|
||||
function huggingFaceFiles(
|
||||
repository: string,
|
||||
revision: string,
|
||||
specs: DownloadFileSpec[]
|
||||
): SpeechModelDownloadFile[] {
|
||||
return specs.map(([name, sizeBytes, sha256]) => ({
|
||||
name,
|
||||
url: `https://huggingface.co/${repository}/resolve/${revision}/${encodeURIComponent(name)}?download=true`,
|
||||
sizeBytes,
|
||||
sha256
|
||||
}))
|
||||
}
|
||||
|
||||
// Why: immutable revisions plus per-file hashes keep direct downloads equivalent to pinned archives.
|
||||
const MODEL_DOWNLOAD_FILES = {
|
||||
'parakeet-tdt-0.6b-v3-int8': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3-int8',
|
||||
'2bda32ec70b097a55adaa07d9a7173915b43cc78',
|
||||
[
|
||||
[
|
||||
'encoder.int8.onnx',
|
||||
652_184_281,
|
||||
'acfc2b4456377e15d04f0243af540b7fe7c992f8d898d751cf134c3a55fd2247'
|
||||
],
|
||||
[
|
||||
'decoder.int8.onnx',
|
||||
11_845_275,
|
||||
'179e50c43d1a9de79c8a24149a2f9bac6eb5981823f2a2ed88d655b24248db4e'
|
||||
],
|
||||
[
|
||||
'joiner.int8.onnx',
|
||||
6_355_277,
|
||||
'3164c13fc2821009440d20fcb5fdc78bff28b4db2f8d0f0b329101719c0948b3'
|
||||
],
|
||||
['tokens.txt', 93_939, 'd58544679ea4bc6ac563d1f545eb7d474bd6cfa467f0a6e2c1dc1c7d37e3c35d']
|
||||
]
|
||||
),
|
||||
'parakeet-tdt-0.6b-v2-int8': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v2-int8',
|
||||
'1ab9323565ddb038682214b292f588070a538ce2',
|
||||
[
|
||||
[
|
||||
'encoder.int8.onnx',
|
||||
652_184_296,
|
||||
'a32b12d17bbbc309d0686fbbcc2987b5e9b8333a7da83fa6b089f0a2acd651ab'
|
||||
],
|
||||
[
|
||||
'decoder.int8.onnx',
|
||||
7_257_753,
|
||||
'b6bb64963457237b900e496ee9994b59294526439fbcc1fecf705b31a15c6b4e'
|
||||
],
|
||||
[
|
||||
'joiner.int8.onnx',
|
||||
1_739_080,
|
||||
'7946164367946e7f9f29a122407c3252b680dbae9a51343eb2488d057c3c43d2'
|
||||
],
|
||||
['tokens.txt', 9_384, 'ec182b70dd42113aff6c5372c75cac58c952443eb22322f57bbd7f53977d497d']
|
||||
]
|
||||
),
|
||||
'zipformer-bilingual-zh-en': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20',
|
||||
'98590b7ed6443e77b714204da2757d75e1a642f4',
|
||||
[
|
||||
[
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
330_083_505,
|
||||
'709f0ed53a734b7942f170127e7547b566cb29c4afc5e67719f314c3d63ccb10'
|
||||
],
|
||||
[
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
13_876_452,
|
||||
'2e3b5ec371f8899ee6acd829fd753ba45772df57a91bdf37cde3136354e7db7d'
|
||||
],
|
||||
[
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
12_833_618,
|
||||
'5f2adc585dd1bec6421c8bb8660d2a73fc8b9ceb24491ef51399ba2a2f0fc31b'
|
||||
],
|
||||
['tokens.txt', 56_317, 'a8e0e4ec53810e433789b54a5c0134a7eaa2ffca595a6334d54c00da858841d3'],
|
||||
['bpe.vocab', 12_564, 'd0b642f3a2eacd5fadefdeff9e0e1358cab729647cbb7fe58cf738e1f7407029']
|
||||
]
|
||||
),
|
||||
'paraformer-bilingual-zh-en': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-streaming-paraformer-bilingual-zh-en',
|
||||
'8e40c43232a1c5c66c82111efc5820d3accca11b',
|
||||
[
|
||||
[
|
||||
'encoder.int8.onnx',
|
||||
165_462_184,
|
||||
'81a70226a8934e6ed92aa1d4fc486b428b5398e2f2619ed4897b7294cab90e9a'
|
||||
],
|
||||
[
|
||||
'decoder.int8.onnx',
|
||||
71_664_561,
|
||||
'f3cca9f77bb9d93c8fcbfb63ae617b6b1ee96818df3aa3b151c40658fe38594f'
|
||||
],
|
||||
['tokens.txt', 75_756, '59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6']
|
||||
]
|
||||
),
|
||||
'zipformer-streaming-en-20m': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-streaming-zipformer-en-20M-2023-02-17',
|
||||
'd42f2d9f7ca24806fb667456a18a9f1b60f70d16',
|
||||
[
|
||||
[
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
88_804_590,
|
||||
'f77a22f4ff94604e1afb2aeb13504d7699363528c047c97d3436087c95c9b659'
|
||||
],
|
||||
[
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
2_092_272,
|
||||
'45a7f940ecfb53d89fa270ad11b88b961e53a317203eb24b1c8e95ed208b0f30'
|
||||
],
|
||||
[
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
1_026_462,
|
||||
'343e17dffa4f386ca206e00d3c406908f68f473c3d35968d6c3cddd5b8559a94'
|
||||
],
|
||||
['tokens.txt', 5_048, '49e3c2646595fd907228b3c6787069658f67b17377c60aeb8619c4551b2316fb']
|
||||
]
|
||||
),
|
||||
'zipformer-streaming-zh-14m': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-streaming-zipformer-zh-14M-2023-02-23',
|
||||
'204ad334e2e683fd295359930cc16fc0432a23ac',
|
||||
[
|
||||
[
|
||||
'encoder-epoch-99-avg-1.onnx',
|
||||
40_948_171,
|
||||
'84c6a8f372686faa5b8f45f2d79f0816f76dcd9f547acb9a90eba2772d7eda8b'
|
||||
],
|
||||
[
|
||||
'decoder-epoch-99-avg-1.onnx',
|
||||
7_509_745,
|
||||
'5ee0f03a2768ff1d5c83ef3a493243c7935d316cd41280037b14783a3467cc78'
|
||||
],
|
||||
[
|
||||
'joiner-epoch-99-avg-1.onnx',
|
||||
7_109_975,
|
||||
'030212efaea9a8b6a4fa98faf6ac6055529c4408cf4865e898220ddd02780f34'
|
||||
],
|
||||
['tokens.txt', 48_697, '8b294db9045d6e5f94647f4c1eec1af4da143a75053c399611444b378ff966ac']
|
||||
]
|
||||
),
|
||||
'zipformer-streaming-korean': huggingFaceFiles(
|
||||
'k2-fsa/sherpa-onnx-streaming-zipformer-korean-2024-06-16',
|
||||
'ba6078bca4daf3f0dd37f79d0ab505af71df14a6',
|
||||
[
|
||||
[
|
||||
'encoder-epoch-99-avg-1.int8.onnx',
|
||||
126_968_852,
|
||||
'8d0b1aa24fbedd4e3948564ab7facd151b8ce9b0c48fc987c541de2de3af5697'
|
||||
],
|
||||
[
|
||||
'decoder-epoch-99-avg-1.int8.onnx',
|
||||
2_844_692,
|
||||
'68ea197936aabd249f38b53a87c775422bca64428ad4427d0e6e8092593e71fb'
|
||||
],
|
||||
[
|
||||
'joiner-epoch-99-avg-1.int8.onnx',
|
||||
2_581_421,
|
||||
'128b80a66a1f718488af8560f9d15895109b99ff3e573f0a0130e03774ef1ced'
|
||||
],
|
||||
['tokens.txt', 60_246, '016bdf0965029263b7ad01b742366ee542ef0bef38261510e8176ff6f2e9e668']
|
||||
]
|
||||
),
|
||||
'parakeet-tdt-ctc-0.6b-ja-int8': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8',
|
||||
'bef18eb066808c90bd0f5df5be685767b0732de8',
|
||||
[
|
||||
[
|
||||
'model.int8.onnx',
|
||||
655_542_604,
|
||||
'3addd00ef5bd1742078389e540b77394e4a508bdf2f4c9ad1b4a76d93e76598e'
|
||||
],
|
||||
['tokens.txt', 28_557, '732f64c53909f2620c713f4106b487d92e6f54a6915b3cd3d1dbd32f9f4f392a']
|
||||
]
|
||||
),
|
||||
'whisper-tiny': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-whisper-tiny',
|
||||
'65176e2deb88badc814a94058666cadccc29b61c',
|
||||
[
|
||||
[
|
||||
'tiny-encoder.onnx',
|
||||
37_647_080,
|
||||
'42c1d4cbf889632ba21ab6f0d4064c80209755f265ce5cd630db4a6793e7089c'
|
||||
],
|
||||
[
|
||||
'tiny-decoder.onnx',
|
||||
114_505_801,
|
||||
'e144c07dc6b55cece24392811f2d934b97013811f5e677d1315d341a0a74a25d'
|
||||
],
|
||||
[
|
||||
'tiny-tokens.txt',
|
||||
816_730,
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
]
|
||||
]
|
||||
),
|
||||
'sense-voice-zh-en-ja-ko-yue': huggingFaceFiles(
|
||||
'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17',
|
||||
'2365baeacb507f821a0c8120fcee3d484dba7a07',
|
||||
[
|
||||
[
|
||||
'model.int8.onnx',
|
||||
239_233_841,
|
||||
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51'
|
||||
],
|
||||
['tokens.txt', 315_894, 'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc']
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
export type DownloadableSpeechModelId = keyof typeof MODEL_DOWNLOAD_FILES
|
||||
|
||||
export function getSpeechModelDownloadMetadata(modelId: DownloadableSpeechModelId): {
|
||||
downloadFiles: SpeechModelDownloadFile[]
|
||||
files: string[]
|
||||
sizeBytes: number
|
||||
} {
|
||||
const downloadFiles = MODEL_DOWNLOAD_FILES[modelId]
|
||||
return {
|
||||
downloadFiles,
|
||||
files: downloadFiles.map(({ name }) => name),
|
||||
sizeBytes: downloadFiles.reduce((total, { sizeBytes }) => total + sizeBytes, 0)
|
||||
}
|
||||
}
|
||||
|
|
@ -19,9 +19,9 @@ vi.mock('electron', () => ({
|
|||
}))
|
||||
|
||||
type ModelManagerInternals = {
|
||||
downloadArchiveWithRetry: (
|
||||
downloadFileWithRetry: (
|
||||
url: string,
|
||||
archivePath: string,
|
||||
filePath: string,
|
||||
expectedSize: number,
|
||||
modelId: string,
|
||||
isAborted: () => boolean,
|
||||
|
|
@ -29,7 +29,7 @@ type ModelManagerInternals = {
|
|||
) => Promise<void>
|
||||
downloadFile: (
|
||||
url: string,
|
||||
archivePath: string,
|
||||
filePath: string,
|
||||
expectedSize: number,
|
||||
modelId: string,
|
||||
isAborted: () => boolean,
|
||||
|
|
@ -141,11 +141,11 @@ describe('ModelManager download resume', () => {
|
|||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -154,13 +154,13 @@ describe('ModelManager download resume', () => {
|
|||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(2)
|
||||
expect(second.sentHeaders.range).toBe('bytes=10-')
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('uses a complete archive after a late transport failure without requesting past EOF', async () => {
|
||||
it('uses a complete file after a late transport failure without requesting past EOF', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
scriptRequest(() => ({
|
||||
|
|
@ -170,11 +170,11 @@ describe('ModelManager download resume', () => {
|
|||
failWith: 'net::ERR_CONNECTION_RESET'
|
||||
}))
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -182,13 +182,13 @@ describe('ModelManager download resume', () => {
|
|||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(1)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('requests the remaining bytes when a clean range response ends before the archive total', async () => {
|
||||
it('requests the remaining bytes when a clean range response ends before the file total', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
const first = scriptRequest((sentHeaders) => {
|
||||
|
|
@ -214,12 +214,12 @@ describe('ModelManager download resume', () => {
|
|||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 10))
|
||||
const filePath = join(dir, 'model.bin')
|
||||
writeFileSync(filePath, PAYLOAD.subarray(0, 10))
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -229,7 +229,7 @@ describe('ModelManager download resume', () => {
|
|||
expect(netRequestMock).toHaveBeenCalledTimes(2)
|
||||
expect(first.sentHeaders.range).toBe('bytes=10-')
|
||||
expect(second.sentHeaders.range).toBe('bytes=15-')
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
|
|
@ -263,12 +263,12 @@ describe('ModelManager download resume', () => {
|
|||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 1))
|
||||
const filePath = join(dir, 'model.bin')
|
||||
writeFileSync(filePath, PAYLOAD.subarray(0, 1))
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -276,7 +276,7 @@ describe('ModelManager download resume', () => {
|
|||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(10)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
|
|
@ -286,20 +286,20 @@ describe('ModelManager download resume', () => {
|
|||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
let bytesWritten = 0
|
||||
// Advances one byte per request against a total larger than the request
|
||||
// ceiling, so it makes forward progress forever without ever completing.
|
||||
const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => {
|
||||
bytesWritten += 1
|
||||
writeFileSync(archivePath, Buffer.alloc(bytesWritten))
|
||||
writeFileSync(filePath, Buffer.alloc(bytesWritten))
|
||||
return Promise.resolve()
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
1_000_000,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -324,18 +324,18 @@ describe('ModelManager download resume', () => {
|
|||
// it takes (regression guard: a fixed failure budget used to abandon a
|
||||
// still-advancing large download around attempt 8).
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
const SLICE = 2
|
||||
let delivered = 0
|
||||
const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockImplementation(() => {
|
||||
delivered = Math.min(delivered + SLICE, PAYLOAD.length)
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, delivered))
|
||||
writeFileSync(filePath, PAYLOAD.subarray(0, delivered))
|
||||
return Promise.reject(new Error('net::ERR_CONNECTION_RESET'))
|
||||
})
|
||||
|
||||
const download = manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
const download = manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -351,14 +351,14 @@ describe('ModelManager download resume', () => {
|
|||
|
||||
await expect(outcome).resolves.toBe('resolved')
|
||||
expect(downloadFileMock).toHaveBeenCalledTimes(PAYLOAD.length / SLICE)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('keeps the known archive total when Content-Range omits it', async () => {
|
||||
it('keeps the known file total when Content-Range omits it', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
scriptRequest((sentHeaders) => {
|
||||
|
|
@ -384,12 +384,12 @@ describe('ModelManager download resume', () => {
|
|||
}
|
||||
})
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 10))
|
||||
const filePath = join(dir, 'model.bin')
|
||||
writeFileSync(filePath, PAYLOAD.subarray(0, 10))
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -397,7 +397,7 @@ describe('ModelManager download resume', () => {
|
|||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(2)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
|
|
@ -415,13 +415,13 @@ describe('ModelManager download resume', () => {
|
|||
chunks: [PAYLOAD.subarray(0, 10)]
|
||||
}))
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, PAYLOAD.subarray(0, 10))
|
||||
const filePath = join(dir, 'model.bin')
|
||||
writeFileSync(filePath, PAYLOAD.subarray(0, 10))
|
||||
|
||||
const error = await manager
|
||||
.downloadFile(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -435,7 +435,7 @@ describe('ModelManager download resume', () => {
|
|||
message: 'Invalid Content-Range for resume at byte 10',
|
||||
retryable: true
|
||||
})
|
||||
expect(existsSync(archivePath)).toBe(false)
|
||||
expect(existsSync(filePath)).toBe(false)
|
||||
expect(mismatched.abortMock).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
|
|
@ -457,11 +457,11 @@ describe('ModelManager download resume', () => {
|
|||
chunks: [PAYLOAD]
|
||||
}))
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
|
||||
await manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
await manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -469,7 +469,7 @@ describe('ModelManager download resume', () => {
|
|||
)
|
||||
|
||||
expect(netRequestMock).toHaveBeenCalledTimes(2)
|
||||
expect(readFileSync(archivePath)).toEqual(PAYLOAD)
|
||||
expect(readFileSync(filePath)).toEqual(PAYLOAD)
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
|
|
@ -482,9 +482,9 @@ describe('ModelManager download resume', () => {
|
|||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
await expect(
|
||||
manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -509,8 +509,8 @@ describe('ModelManager download resume', () => {
|
|||
|
||||
const error = await manager
|
||||
.downloadFile(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false
|
||||
|
|
@ -529,7 +529,7 @@ describe('ModelManager download resume', () => {
|
|||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-resume-'))
|
||||
try {
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
const filePath = join(dir, 'model.bin')
|
||||
const rateLimitError = Object.assign(new Error('HTTP 429'), {
|
||||
httpStatusCode: 429,
|
||||
retryAfterMs: 3_000
|
||||
|
|
@ -538,12 +538,12 @@ describe('ModelManager download resume', () => {
|
|||
.spyOn(manager, 'downloadFile')
|
||||
.mockRejectedValueOnce(rateLimitError)
|
||||
.mockImplementationOnce(() => {
|
||||
writeFileSync(archivePath, PAYLOAD)
|
||||
writeFileSync(filePath, PAYLOAD)
|
||||
return Promise.resolve()
|
||||
})
|
||||
const download = manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
archivePath,
|
||||
const download = manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
filePath,
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -572,9 +572,9 @@ describe('ModelManager download resume', () => {
|
|||
const downloadFileMock = vi.spyOn(manager, 'downloadFile').mockRejectedValue(rateLimitError)
|
||||
|
||||
await expect(
|
||||
manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -597,9 +597,9 @@ describe('ModelManager download resume', () => {
|
|||
.spyOn(manager, 'downloadFile')
|
||||
.mockRejectedValue(new Error('net::ERR_CONNECTION_RESET'))
|
||||
|
||||
const download = manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
const download = manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
@ -645,9 +645,9 @@ describe('ModelManager download resume', () => {
|
|||
const controller = new AbortController()
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
const download = manager.downloadArchiveWithRetry(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
const download = manager.downloadFileWithRetry(
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
PAYLOAD.length,
|
||||
'm',
|
||||
() => false,
|
||||
|
|
|
|||
|
|
@ -64,8 +64,8 @@ describe('ModelManager stream cleanup', () => {
|
|||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
const download = manager.downloadFile(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
4,
|
||||
'm',
|
||||
() => false
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
|
@ -84,13 +84,16 @@ describe('ModelManager Windows model path handling', () => {
|
|||
appGetPathMock.mockImplementation((name: string) =>
|
||||
name === 'userData' ? userDataDir : join(dir, name)
|
||||
)
|
||||
const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local')
|
||||
const manifest = SPEECH_MODEL_CATALOG.find(
|
||||
(model) => model.id === 'zipformer-streaming-zh-14m'
|
||||
)
|
||||
expect(manifest?.files).toBeDefined()
|
||||
const legacyModelDir = join(userDataDir, 'speech-models', manifest!.id)
|
||||
for (const file of manifest!.files ?? []) {
|
||||
const filePath = join(legacyModelDir, file)
|
||||
for (const file of manifest!.downloadFiles ?? []) {
|
||||
const filePath = join(legacyModelDir, file.name)
|
||||
mkdirSync(dirname(filePath), { recursive: true })
|
||||
writeFileSync(filePath, 'model file')
|
||||
writeFileSync(filePath, '')
|
||||
truncateSync(filePath, file.sizeBytes)
|
||||
}
|
||||
|
||||
const manager = new ModelManager()
|
||||
|
|
@ -120,12 +123,15 @@ describe('ModelManager Windows model path handling', () => {
|
|||
appGetPathMock.mockImplementation((name: string) =>
|
||||
name === 'userData' ? userDataDir : join(dir, name)
|
||||
)
|
||||
const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local')
|
||||
const manifest = SPEECH_MODEL_CATALOG.find(
|
||||
(model) => model.id === 'zipformer-streaming-zh-14m'
|
||||
)
|
||||
const legacyModelDir = join(userDataDir, 'speech-models', manifest!.id)
|
||||
for (const file of manifest!.files ?? []) {
|
||||
const filePath = join(legacyModelDir, file)
|
||||
for (const file of manifest!.downloadFiles ?? []) {
|
||||
const filePath = join(legacyModelDir, file.name)
|
||||
mkdirSync(dirname(filePath), { recursive: true })
|
||||
writeFileSync(filePath, 'model file')
|
||||
writeFileSync(filePath, '')
|
||||
truncateSync(filePath, file.sizeBytes)
|
||||
}
|
||||
|
||||
const manager = new ModelManager()
|
||||
|
|
|
|||
|
|
@ -1,15 +1,14 @@
|
|||
import { createHash } from 'node:crypto'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { existsSync, mkdirSync, mkdtempSync, rmSync, truncateSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, join } from 'node:path'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { SPEECH_MODEL_CATALOG } from './model-catalog'
|
||||
import { ModelManager } from './model-manager'
|
||||
|
||||
const { hasOpenAiSpeechApiKeyMock, netRequestMock, spawnMock } = vi.hoisted(() => ({
|
||||
const { hasOpenAiSpeechApiKeyMock, netRequestMock } = vi.hoisted(() => ({
|
||||
hasOpenAiSpeechApiKeyMock: vi.fn(),
|
||||
netRequestMock: vi.fn(),
|
||||
spawnMock: vi.fn()
|
||||
netRequestMock: vi.fn()
|
||||
}))
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
|
|
@ -21,17 +20,22 @@ vi.mock('electron', () => ({
|
|||
}
|
||||
}))
|
||||
|
||||
vi.mock('child_process', async () => {
|
||||
const actual = await vi.importActual('child_process')
|
||||
return { ...(actual as Record<string, unknown>), spawn: spawnMock }
|
||||
})
|
||||
|
||||
vi.mock('./openai-api-key-store', () => ({
|
||||
hasOpenAiSpeechApiKey: hasOpenAiSpeechApiKeyMock
|
||||
}))
|
||||
|
||||
type ModelManagerInternals = {
|
||||
verifyArchiveSha256: (archivePath: string, expectedSha256: string) => Promise<void>
|
||||
verifyFileSha256: (filePath: string, expectedSha256: string) => Promise<void>
|
||||
downloadFileWithRetry: (
|
||||
url: string,
|
||||
filePath: string,
|
||||
expectedSize: number,
|
||||
modelId: string,
|
||||
isAborted: () => boolean,
|
||||
signal: AbortSignal,
|
||||
completedBytes?: number,
|
||||
modelTotalBytes?: number
|
||||
) => Promise<void>
|
||||
downloadFile: (
|
||||
url: string,
|
||||
dest: string,
|
||||
|
|
@ -40,12 +44,6 @@ type ModelManagerInternals = {
|
|||
isAborted: () => boolean,
|
||||
signal?: AbortSignal
|
||||
) => Promise<void>
|
||||
extractArchive: (
|
||||
archivePath: string,
|
||||
destDir: string,
|
||||
modelId: string,
|
||||
isAborted: () => boolean
|
||||
) => Promise<void>
|
||||
}
|
||||
|
||||
describe('ModelManager', () => {
|
||||
|
|
@ -53,28 +51,38 @@ describe('ModelManager', () => {
|
|||
netRequestMock.mockReset()
|
||||
hasOpenAiSpeechApiKeyMock.mockReset()
|
||||
hasOpenAiSpeechApiKeyMock.mockReturnValue(false)
|
||||
spawnMock.mockReset()
|
||||
})
|
||||
|
||||
it('requires pinned SHA-256 hashes for every catalog archive', () => {
|
||||
it('requires pinned, internally consistent metadata for every model file', () => {
|
||||
for (const manifest of SPEECH_MODEL_CATALOG) {
|
||||
if (manifest.provider !== 'local') {
|
||||
continue
|
||||
}
|
||||
expect(manifest.archiveSha256).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(manifest.downloadFiles?.length).toBeGreaterThan(0)
|
||||
expect(manifest.files).toEqual(manifest.downloadFiles?.map(({ name }) => name))
|
||||
expect(manifest.sizeBytes).toBe(
|
||||
manifest.downloadFiles?.reduce((total, { sizeBytes }) => total + sizeBytes, 0)
|
||||
)
|
||||
for (const file of manifest.downloadFiles ?? []) {
|
||||
expect(file.url).toMatch(
|
||||
/^https:\/\/huggingface\.co\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\//
|
||||
)
|
||||
expect(file.sha256).toMatch(/^[a-f0-9]{64}$/)
|
||||
expect(file.sizeBytes).toBeGreaterThan(0)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('verifies downloaded archive hashes before extraction', async () => {
|
||||
it('verifies downloaded model file hashes before installation', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-'))
|
||||
try {
|
||||
const archivePath = join(dir, 'model.tar.bz2')
|
||||
writeFileSync(archivePath, 'known archive bytes')
|
||||
const expected = createHash('sha256').update('known archive bytes').digest('hex')
|
||||
const filePath = join(dir, 'model.onnx')
|
||||
writeFileSync(filePath, 'known model bytes')
|
||||
const expected = createHash('sha256').update('known model bytes').digest('hex')
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
await expect(manager.verifyArchiveSha256(archivePath, expected)).resolves.toBeUndefined()
|
||||
await expect(manager.verifyArchiveSha256(archivePath, '0'.repeat(64))).rejects.toThrow(
|
||||
await expect(manager.verifyFileSha256(filePath, expected)).resolves.toBeUndefined()
|
||||
await expect(manager.verifyFileSha256(filePath, '0'.repeat(64))).rejects.toThrow(
|
||||
/integrity verification/
|
||||
)
|
||||
} finally {
|
||||
|
|
@ -89,8 +97,8 @@ describe('ModelManager', () => {
|
|||
|
||||
await expect(
|
||||
manager.downloadFile(
|
||||
'http://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
'http://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
1,
|
||||
'm',
|
||||
() => false
|
||||
|
|
@ -101,6 +109,46 @@ describe('ModelManager', () => {
|
|||
}
|
||||
})
|
||||
|
||||
it('installs individually verified model files through a staging directory', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-'))
|
||||
try {
|
||||
const manifest = SPEECH_MODEL_CATALOG.find(
|
||||
(model) => model.id === 'zipformer-streaming-zh-14m'
|
||||
)!
|
||||
const manager = new ModelManager(dir)
|
||||
const internals = manager as unknown as ModelManagerInternals
|
||||
const downloadMock = vi
|
||||
.spyOn(internals, 'downloadFileWithRetry')
|
||||
.mockImplementation(async (_url, filePath, expectedSize) => {
|
||||
writeFileSync(filePath, '')
|
||||
truncateSync(filePath, expectedSize)
|
||||
})
|
||||
const verifyMock = vi.spyOn(internals, 'verifyFileSha256').mockResolvedValue()
|
||||
|
||||
await manager.downloadModel(manifest.id)
|
||||
|
||||
const modelDir = manager.getModelDir(manifest.id)
|
||||
expect(downloadMock).toHaveBeenCalledTimes(manifest.downloadFiles?.length ?? 0)
|
||||
expect(verifyMock).toHaveBeenCalledTimes(manifest.downloadFiles?.length ?? 0)
|
||||
let expectedOffset = 0
|
||||
for (const [index, file] of (manifest.downloadFiles ?? []).entries()) {
|
||||
expect(downloadMock.mock.calls[index]?.slice(6)).toEqual([
|
||||
expectedOffset,
|
||||
manifest.sizeBytes
|
||||
])
|
||||
expectedOffset += file.sizeBytes
|
||||
expect(existsSync(join(modelDir, file.name))).toBe(true)
|
||||
}
|
||||
expect(existsSync(`${modelDir}.partial`)).toBe(false)
|
||||
await expect(manager.getModelState(manifest.id)).resolves.toEqual({
|
||||
id: manifest.id,
|
||||
status: 'ready'
|
||||
})
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('marks OpenAI transcription models ready only when an API key is configured', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-'))
|
||||
try {
|
||||
|
|
@ -125,14 +173,17 @@ describe('ModelManager', () => {
|
|||
it('deletes a ready local model and reports it as not downloaded', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-'))
|
||||
try {
|
||||
const manifest = SPEECH_MODEL_CATALOG.find((model) => model.provider === 'local')
|
||||
const manifest = SPEECH_MODEL_CATALOG.find(
|
||||
(model) => model.id === 'zipformer-streaming-zh-14m'
|
||||
)
|
||||
expect(manifest?.files).toBeDefined()
|
||||
const manager = new ModelManager(dir)
|
||||
const modelDir = manager.getModelDir(manifest!.id)
|
||||
for (const file of manifest!.files ?? []) {
|
||||
const path = join(modelDir, file)
|
||||
for (const file of manifest!.downloadFiles ?? []) {
|
||||
const path = join(modelDir, file.name)
|
||||
mkdirSync(dirname(path), { recursive: true })
|
||||
writeFileSync(path, 'model file')
|
||||
writeFileSync(path, '')
|
||||
truncateSync(path, file.sizeBytes)
|
||||
}
|
||||
|
||||
await expect(manager.getModelState(manifest!.id)).resolves.toEqual({
|
||||
|
|
@ -288,8 +339,8 @@ describe('ModelManager', () => {
|
|||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
const download = manager.downloadFile(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
1,
|
||||
'm',
|
||||
() => true,
|
||||
|
|
@ -370,8 +421,8 @@ describe('ModelManager', () => {
|
|||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
const download = manager.downloadFile(
|
||||
'https://example.com/model.tar.bz2',
|
||||
join(dir, 'model.tar.bz2'),
|
||||
'https://example.com/model.bin',
|
||||
join(dir, 'model.bin'),
|
||||
1,
|
||||
'm',
|
||||
() => false
|
||||
|
|
@ -397,59 +448,4 @@ describe('ModelManager', () => {
|
|||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
it('clears extraction abort polling when the child does not close', async () => {
|
||||
vi.useFakeTimers()
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-model-manager-'))
|
||||
try {
|
||||
const handlers: Record<string, ((arg?: unknown) => void)[]> = {
|
||||
close: [],
|
||||
error: []
|
||||
}
|
||||
const stderrHandlers: ((chunk: Buffer) => void)[] = []
|
||||
const child = {
|
||||
stderr: {
|
||||
on: vi.fn((_event: string, cb: (chunk: Buffer) => void) => {
|
||||
stderrHandlers.push(cb)
|
||||
return child.stderr
|
||||
}),
|
||||
off: vi.fn((_event: string, cb: (chunk: Buffer) => void) => {
|
||||
const index = stderrHandlers.indexOf(cb)
|
||||
if (index !== -1) {
|
||||
stderrHandlers.splice(index, 1)
|
||||
}
|
||||
return child.stderr
|
||||
})
|
||||
},
|
||||
kill: vi.fn(),
|
||||
on: vi.fn((event: string, cb: (arg?: unknown) => void) => {
|
||||
handlers[event]?.push(cb)
|
||||
return child
|
||||
}),
|
||||
off: vi.fn((event: string, cb: (arg?: unknown) => void) => {
|
||||
handlers[event] = handlers[event]?.filter((handler) => handler !== cb) ?? []
|
||||
return child
|
||||
})
|
||||
}
|
||||
spawnMock.mockReturnValue(child)
|
||||
const manager = new ModelManager(dir) as unknown as ModelManagerInternals
|
||||
|
||||
const extraction = manager.extractArchive(join(dir, 'model.tar.bz2'), dir, 'm', () => true)
|
||||
const rejection = expect(extraction).rejects.toThrow('Aborted')
|
||||
await vi.advanceTimersByTimeAsync(250)
|
||||
await rejection
|
||||
|
||||
expect(child.kill).toHaveBeenCalledWith('SIGKILL')
|
||||
expect(child.kill).toHaveBeenCalledTimes(1)
|
||||
expect(handlers.close).toHaveLength(0)
|
||||
expect(handlers.error).toHaveLength(0)
|
||||
expect(stderrHandlers).toHaveLength(0)
|
||||
|
||||
vi.advanceTimersByTime(1000)
|
||||
expect(child.kill).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
rmSync(dir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
/* eslint-disable max-lines -- Why: model download, checksum, extraction, and cleanup share one state machine so progress/error transitions stay coupled. */
|
||||
/* eslint-disable max-lines -- Why: model download, checksum, retry, and cleanup share one state machine so progress/error transitions stay coupled. */
|
||||
import { app, net } from 'electron'
|
||||
import { join, resolve, relative } from 'node:path'
|
||||
import {
|
||||
|
|
@ -9,10 +9,9 @@ import {
|
|||
rmSync,
|
||||
statSync
|
||||
} from 'node:fs'
|
||||
import { readdir, rm } from 'node:fs/promises'
|
||||
import { rename, rm } from 'node:fs/promises'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import { spawn } from 'node:child_process'
|
||||
import type {
|
||||
SpeechModelManifest,
|
||||
SpeechModelState,
|
||||
|
|
@ -20,7 +19,6 @@ import type {
|
|||
} from '../../shared/speech-types'
|
||||
import { SPEECH_MODEL_CATALOG, getCatalogModel, isLocalSpeechModel } from './model-catalog'
|
||||
import { hasOpenAiSpeechApiKey } from './openai-api-key-store'
|
||||
import { resolveTarExecutable } from './tar-executable'
|
||||
import {
|
||||
getSpeechModelCacheDirCandidates,
|
||||
migrateSpeechModelCacheIfNeeded,
|
||||
|
|
@ -42,7 +40,11 @@ type HttpStatusError = Error & {
|
|||
retryAfterMs?: number
|
||||
retryable?: boolean
|
||||
}
|
||||
type DownloadTotals = { totalBytes: number }
|
||||
type DownloadTotals = {
|
||||
totalBytes: number
|
||||
completedBytes: number
|
||||
modelTotalBytes: number
|
||||
}
|
||||
type ContentRange = { start: number; end: number; totalBytes?: number }
|
||||
|
||||
const DOWNLOAD_IDLE_TIMEOUT_MS = 120_000
|
||||
|
|
@ -254,10 +256,16 @@ export class ModelManager {
|
|||
}
|
||||
|
||||
private validateModelFiles(manifest: SpeechModelManifest, modelDir: string): boolean {
|
||||
if (!manifest.files) {
|
||||
if (!manifest.downloadFiles) {
|
||||
return false
|
||||
}
|
||||
return manifest.files.every((f) => existsSync(join(modelDir, f)))
|
||||
return manifest.downloadFiles.every(({ name, sizeBytes }) => {
|
||||
try {
|
||||
return statSync(join(modelDir, name)).size === sizeBytes
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async downloadModel(modelId: string): Promise<void> {
|
||||
|
|
@ -273,7 +281,7 @@ export class ModelManager {
|
|||
if (!isLocalSpeechModel(manifest)) {
|
||||
throw new Error(`Model does not support downloads: ${modelId}`)
|
||||
}
|
||||
if (!manifest.downloadUrl || !manifest.archiveSha256 || !manifest.sizeBytes) {
|
||||
if (!manifest.downloadFiles?.length || !manifest.sizeBytes) {
|
||||
throw new Error(`Model download metadata missing: ${modelId}`)
|
||||
}
|
||||
|
||||
|
|
@ -285,15 +293,16 @@ export class ModelManager {
|
|||
|
||||
this.updateState(modelId, 'downloading', 0)
|
||||
|
||||
const archivePath = join(this.modelsDir, `${modelId}.tar.bz2`)
|
||||
// Why: resume appends, so a leftover archive from a crashed run would corrupt the download.
|
||||
const stagingDir = `${modelDir}.partial`
|
||||
const legacyArchivePath = join(this.modelsDir, `${modelId}.tar.bz2`)
|
||||
// Why: resuming an unverified file left by a crashed process could preserve corrupt bytes.
|
||||
rmSync(stagingDir, { recursive: true, force: true })
|
||||
try {
|
||||
if (existsSync(archivePath)) {
|
||||
rmSync(archivePath)
|
||||
}
|
||||
rmSync(legacyArchivePath, { force: true })
|
||||
} catch {
|
||||
// best-effort; the first (non-resumed) attempt truncates on write
|
||||
// best-effort legacy cleanup
|
||||
}
|
||||
mkdirSync(stagingDir, { recursive: true })
|
||||
let aborted = false
|
||||
const abortController = new AbortController()
|
||||
|
||||
|
|
@ -307,69 +316,34 @@ export class ModelManager {
|
|||
this.activeDownloads.set(modelId, handle)
|
||||
|
||||
try {
|
||||
await this.downloadArchiveWithRetry(
|
||||
manifest.downloadUrl,
|
||||
archivePath,
|
||||
manifest.sizeBytes,
|
||||
await this.downloadModelFiles(
|
||||
manifest,
|
||||
stagingDir,
|
||||
modelId,
|
||||
() => aborted,
|
||||
abortController.signal
|
||||
)
|
||||
|
||||
if (aborted) {
|
||||
this.cleanup(modelId, archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
await this.verifyArchiveSha256(archivePath, manifest.archiveSha256)
|
||||
|
||||
if (aborted) {
|
||||
this.cleanup(modelId, archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
this.updateState(modelId, 'extracting')
|
||||
await this.extractArchive(archivePath, this.modelsDir, modelId, () => aborted)
|
||||
|
||||
if (aborted) {
|
||||
this.cleanup(modelId, archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
if (!this.validateModelFiles(manifest, modelDir)) {
|
||||
// Why: some archives nest files in a subdir; scan one level down and move them up.
|
||||
await this.flattenNestedDir(modelDir, manifest)
|
||||
}
|
||||
|
||||
if (aborted) {
|
||||
this.cleanup(modelId, archivePath)
|
||||
return
|
||||
}
|
||||
|
||||
if (!this.validateModelFiles(manifest, modelDir)) {
|
||||
throw new Error('Model files missing after extraction')
|
||||
}
|
||||
|
||||
await rm(modelDir, { recursive: true, force: true })
|
||||
await rename(stagingDir, modelDir)
|
||||
this.updateState(modelId, 'ready')
|
||||
} catch (err) {
|
||||
if (!aborted) {
|
||||
console.error('[speech] Model download failed:', modelId, err)
|
||||
this.updateState(modelId, 'error', undefined, String(err))
|
||||
}
|
||||
this.cleanup(modelId, archivePath)
|
||||
this.removeModelDownloadFiles(modelDir, stagingDir, legacyArchivePath)
|
||||
if (!aborted) {
|
||||
// Why: the settings UI awaits this to surface failures; stay quiet on cancellation, rethrow real errors.
|
||||
throw err
|
||||
}
|
||||
} finally {
|
||||
this.activeDownloads.delete(modelId)
|
||||
try {
|
||||
if (existsSync(archivePath)) {
|
||||
rmSync(archivePath)
|
||||
}
|
||||
} catch {
|
||||
// best-effort archive cleanup
|
||||
}
|
||||
this.removeModelDownloadStaging(stagingDir, legacyArchivePath)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -395,6 +369,8 @@ export class ModelManager {
|
|||
if (existsSync(modelDir)) {
|
||||
await rm(modelDir, { recursive: true, force: true })
|
||||
}
|
||||
await rm(`${modelDir}.partial`, { recursive: true, force: true })
|
||||
await rm(join(this.modelsDir, `${modelId}.tar.bz2`), { force: true })
|
||||
// Why: also delete the pre-migration copy, or the next launch re-migrates it and resurrects the model.
|
||||
if (this.migrationSourceDir) {
|
||||
const sourceModelDir = this.getSafeModelDir(modelId, this.migrationSourceDir)
|
||||
|
|
@ -420,28 +396,75 @@ export class ModelManager {
|
|||
}
|
||||
}
|
||||
|
||||
private getPartialArchiveBytes(archivePath: string): number {
|
||||
private async downloadModelFiles(
|
||||
manifest: SpeechModelManifest,
|
||||
stagingDir: string,
|
||||
modelId: string,
|
||||
isAborted: () => boolean,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
if (!manifest.downloadFiles?.length || !manifest.sizeBytes) {
|
||||
throw new Error(`Model download metadata missing: ${modelId}`)
|
||||
}
|
||||
|
||||
let completedBytes = 0
|
||||
for (const file of manifest.downloadFiles) {
|
||||
if (
|
||||
!file.name ||
|
||||
file.name === '.' ||
|
||||
file.name === '..' ||
|
||||
file.name.includes('/') ||
|
||||
file.name.includes('\\')
|
||||
) {
|
||||
throw new Error(`Invalid model download filename: ${file.name}`)
|
||||
}
|
||||
const filePath = join(stagingDir, file.name)
|
||||
await this.downloadFileWithRetry(
|
||||
file.url,
|
||||
filePath,
|
||||
file.sizeBytes,
|
||||
modelId,
|
||||
isAborted,
|
||||
signal,
|
||||
completedBytes,
|
||||
manifest.sizeBytes
|
||||
)
|
||||
if (isAborted()) {
|
||||
return
|
||||
}
|
||||
await this.verifyFileSha256(filePath, file.sha256)
|
||||
completedBytes += file.sizeBytes
|
||||
}
|
||||
|
||||
if (!this.validateModelFiles(manifest, stagingDir)) {
|
||||
throw new Error('Model files missing after download')
|
||||
}
|
||||
}
|
||||
|
||||
private getPartialDownloadBytes(filePath: string): number {
|
||||
try {
|
||||
return statSync(archivePath).size
|
||||
return statSync(filePath).size
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadArchiveWithRetry(
|
||||
private async downloadFileWithRetry(
|
||||
url: string,
|
||||
archivePath: string,
|
||||
filePath: string,
|
||||
expectedSize: number,
|
||||
modelId: string,
|
||||
isAborted: () => boolean,
|
||||
signal: AbortSignal
|
||||
signal: AbortSignal,
|
||||
completedBytes = 0,
|
||||
modelTotalBytes = expectedSize
|
||||
): Promise<void> {
|
||||
let requestCount = 0
|
||||
let noProgressStreak = 0
|
||||
const totals: DownloadTotals = { totalBytes: expectedSize }
|
||||
const totals: DownloadTotals = { totalBytes: expectedSize, completedBytes, modelTotalBytes }
|
||||
for (;;) {
|
||||
requestCount += 1
|
||||
const offset = this.getPartialArchiveBytes(archivePath)
|
||||
const offset = this.getPartialDownloadBytes(filePath)
|
||||
// Why: transport can fail after the last byte hits disk; the SHA-256 check is the real completion test.
|
||||
if (offset === totals.totalBytes) {
|
||||
return
|
||||
|
|
@ -459,7 +482,7 @@ export class ModelManager {
|
|||
// Why: restart from the canonical URL, not the last redirect, because signed CDN redirect URLs expire.
|
||||
await this.downloadFile(
|
||||
url,
|
||||
archivePath,
|
||||
filePath,
|
||||
expectedSize,
|
||||
modelId,
|
||||
isAborted,
|
||||
|
|
@ -468,7 +491,7 @@ export class ModelManager {
|
|||
offset,
|
||||
totals
|
||||
)
|
||||
const receivedBytes = this.getPartialArchiveBytes(archivePath)
|
||||
const receivedBytes = this.getPartialDownloadBytes(filePath)
|
||||
if (receivedBytes === totals.totalBytes) {
|
||||
return
|
||||
}
|
||||
|
|
@ -492,7 +515,7 @@ export class ModelManager {
|
|||
if (isAborted() || signal.aborted) {
|
||||
throw err
|
||||
}
|
||||
const receivedBytes = this.getPartialArchiveBytes(archivePath)
|
||||
const receivedBytes = this.getPartialDownloadBytes(filePath)
|
||||
if (receivedBytes === totals.totalBytes) {
|
||||
return
|
||||
}
|
||||
|
|
@ -666,7 +689,7 @@ export class ModelManager {
|
|||
(parsedLength <= 0 || parsedLength === contentRange.end - contentRange.start + 1)
|
||||
|
||||
if (resumeOffset > 0 && response.statusCode === 206 && !resumed) {
|
||||
// Why: appending an unverified range can silently corrupt the archive; discard and retry from byte zero.
|
||||
// Why: appending an unverified range can silently corrupt the file; discard and retry from byte zero.
|
||||
try {
|
||||
rmSync(dest)
|
||||
} catch {
|
||||
|
|
@ -728,7 +751,11 @@ export class ModelManager {
|
|||
return
|
||||
}
|
||||
downloaded += chunk.length
|
||||
const progress = Math.min(0.9, (progressBase + downloaded) / totalSize)
|
||||
const progress = Math.min(
|
||||
0.9,
|
||||
((totals?.completedBytes ?? 0) + progressBase + downloaded) /
|
||||
(totals?.modelTotalBytes ?? totalSize)
|
||||
)
|
||||
this.updateState(modelId, 'downloading', progress)
|
||||
}
|
||||
|
||||
|
|
@ -765,10 +792,10 @@ export class ModelManager {
|
|||
})
|
||||
}
|
||||
|
||||
private verifyArchiveSha256(archivePath: string, expectedSha256: string): Promise<void> {
|
||||
private verifyFileSha256(filePath: string, expectedSha256: string): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const hash = createHash('sha256')
|
||||
const stream = createReadStream(archivePath)
|
||||
const stream = createReadStream(filePath)
|
||||
let settled = false
|
||||
|
||||
const cleanup = (): void => {
|
||||
|
|
@ -801,8 +828,8 @@ export class ModelManager {
|
|||
const onEnd = (): void => {
|
||||
const actualSha256 = hash.digest('hex')
|
||||
if (actualSha256 !== expectedSha256.toLowerCase()) {
|
||||
// Why: archives feed native parsers, so verify contents against compromised/redirected release assets.
|
||||
settleReject(new Error('Downloaded model archive failed integrity verification'))
|
||||
// Why: model artifacts feed native runtimes, so verify every downloaded file before installation.
|
||||
settleReject(new Error('Downloaded model file failed integrity verification'))
|
||||
return
|
||||
}
|
||||
settleResolve()
|
||||
|
|
@ -814,125 +841,24 @@ export class ModelManager {
|
|||
})
|
||||
}
|
||||
|
||||
private extractArchive(
|
||||
archivePath: string,
|
||||
destDir: string,
|
||||
modelId: string,
|
||||
isAborted: () => boolean
|
||||
): Promise<void> {
|
||||
const modelDir = join(destDir, modelId)
|
||||
mkdirSync(modelDir, { recursive: true })
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
// Why: spawn (not exec) so slow bzip2 stderr can't overflow exec's 1MB maxBuffer and silently kill the process.
|
||||
const tarExecutable = resolveTarExecutable()
|
||||
const child = spawn(
|
||||
tarExecutable,
|
||||
['-xjf', archivePath, '-C', modelDir, '--strip-components=1'],
|
||||
{
|
||||
stdio: ['ignore', 'ignore', 'pipe'],
|
||||
windowsHide: true
|
||||
}
|
||||
)
|
||||
|
||||
let stderr = ''
|
||||
let settled = false
|
||||
let timeout: ReturnType<typeof setTimeout> | null = null
|
||||
let abortPoll: ReturnType<typeof setInterval> | null = null
|
||||
const cleanup = (): void => {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout)
|
||||
timeout = null
|
||||
}
|
||||
if (abortPoll) {
|
||||
clearInterval(abortPoll)
|
||||
abortPoll = null
|
||||
}
|
||||
child.stderr?.off('data', onStderrData)
|
||||
child.off('close', onClose)
|
||||
child.off('error', onError)
|
||||
}
|
||||
const fail = (error: Error, killChild = false): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
if (killChild) {
|
||||
child.kill('SIGKILL')
|
||||
}
|
||||
reject(error)
|
||||
}
|
||||
const onStderrData = (chunk: Buffer): void => {
|
||||
stderr += chunk.toString()
|
||||
}
|
||||
const onClose = (code: number | null): void => {
|
||||
if (settled) {
|
||||
return
|
||||
}
|
||||
settled = true
|
||||
cleanup()
|
||||
if (code === 0) {
|
||||
resolve()
|
||||
} else {
|
||||
reject(new Error(`tar exited with code ${code}: ${stderr.slice(0, 500)}`))
|
||||
}
|
||||
}
|
||||
const onError = (err: Error): void => {
|
||||
fail(err)
|
||||
}
|
||||
|
||||
child.stderr?.on('data', onStderrData)
|
||||
timeout = setTimeout(() => {
|
||||
fail(new Error('Extraction timed out after 10 minutes'), true)
|
||||
}, 600_000)
|
||||
abortPoll = setInterval(() => {
|
||||
if (isAborted()) {
|
||||
// Why: a wedged child may never emit close/error, so abort must kill it here.
|
||||
fail(new Error('Aborted'), true)
|
||||
}
|
||||
}, 250)
|
||||
|
||||
child.on('close', onClose)
|
||||
child.on('error', onError)
|
||||
})
|
||||
}
|
||||
|
||||
private async flattenNestedDir(modelDir: string, manifest: SpeechModelManifest): Promise<void> {
|
||||
if (!manifest.files) {
|
||||
return
|
||||
}
|
||||
const entries = await readdir(modelDir, { withFileTypes: true })
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) {
|
||||
const nestedDir = join(modelDir, entry.name)
|
||||
const nestedFiles = await readdir(nestedDir)
|
||||
const hasExpected = manifest.files.some((f) => nestedFiles.includes(f))
|
||||
if (hasExpected) {
|
||||
const { rename: fsRename } = await import('node:fs/promises')
|
||||
for (const file of nestedFiles) {
|
||||
await fsRename(join(nestedDir, file), join(modelDir, file))
|
||||
}
|
||||
await rm(nestedDir, { recursive: true, force: true })
|
||||
return
|
||||
}
|
||||
private removeModelDownloadStaging(stagingDir: string, legacyArchivePath: string): void {
|
||||
for (const path of [stagingDir, legacyArchivePath]) {
|
||||
try {
|
||||
rmSync(path, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private cleanup(modelId: string, archivePath: string): void {
|
||||
private removeModelDownloadFiles(
|
||||
modelDir: string,
|
||||
stagingDir: string,
|
||||
legacyArchivePath: string
|
||||
): void {
|
||||
this.removeModelDownloadStaging(stagingDir, legacyArchivePath)
|
||||
try {
|
||||
if (existsSync(archivePath)) {
|
||||
rmSync(archivePath)
|
||||
}
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
const modelDir = this.getModelDir(modelId)
|
||||
try {
|
||||
if (existsSync(modelDir)) {
|
||||
rmSync(modelDir, { recursive: true })
|
||||
}
|
||||
rmSync(modelDir, { recursive: true, force: true })
|
||||
} catch {
|
||||
// best-effort
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,9 +26,7 @@ export function resolveTokens(files: string[], modelDir: string): string {
|
|||
return join(modelDir, match)
|
||||
}
|
||||
|
||||
// Why: BPE models need a vocab file for hotwords token matching. The file
|
||||
// ships in the model archive but isn't listed in the manifest. We discover
|
||||
// it at runtime to avoid breaking existing downloads.
|
||||
// Why: BPE models need a vocab file for hotwords token matching, but older caches may omit it.
|
||||
function discoverBpeVocab(modelDir: string): string | undefined {
|
||||
try {
|
||||
const entries = readdirSync(modelDir)
|
||||
|
|
|
|||
|
|
@ -1,27 +0,0 @@
|
|||
import { existsSync } from 'node:fs'
|
||||
import { win32 as pathWin32 } from 'node:path'
|
||||
|
||||
export function resolveTarExecutable(
|
||||
options: {
|
||||
platform?: NodeJS.Platform
|
||||
env?: NodeJS.ProcessEnv
|
||||
exists?: (path: string) => boolean
|
||||
} = {}
|
||||
): string {
|
||||
const platform = options.platform ?? process.platform
|
||||
if (platform !== 'win32') {
|
||||
return 'tar'
|
||||
}
|
||||
|
||||
const env = options.env ?? process.env
|
||||
const systemRoot = env.SystemRoot ?? env.WINDIR ?? 'C:\\Windows'
|
||||
const candidate = pathWin32.join(systemRoot, 'System32', 'tar.exe')
|
||||
const exists = options.exists ?? existsSync
|
||||
if (exists(candidate)) {
|
||||
return candidate
|
||||
}
|
||||
|
||||
// Why: packaged Windows apps can have a stripped PATH. Use the OS tar
|
||||
// location explicitly, and fail with a repairable error if it is absent.
|
||||
throw new Error(`Windows tar.exe not found at ${candidate}`)
|
||||
}
|
||||
|
|
@ -9,6 +9,13 @@ export type SpeechModelProvider = 'local' | 'openai'
|
|||
|
||||
export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe'
|
||||
|
||||
export type SpeechModelDownloadFile = {
|
||||
name: string
|
||||
url: string
|
||||
sizeBytes: number
|
||||
sha256: string
|
||||
}
|
||||
|
||||
export type SpeechModelManifest = {
|
||||
id: string
|
||||
label: string
|
||||
|
|
@ -17,9 +24,7 @@ export type SpeechModelManifest = {
|
|||
provider: SpeechModelProvider
|
||||
language: string
|
||||
sizeBytes?: number
|
||||
downloadUrl?: string
|
||||
archiveSha256?: string
|
||||
archiveFormat?: 'tar.bz2'
|
||||
downloadFiles?: SpeechModelDownloadFile[]
|
||||
files?: string[]
|
||||
sampleRate: number
|
||||
streaming: boolean
|
||||
|
|
|
|||
Loading…
Reference in New Issue