feat(speech): add Parakeet TDT-CTC 0.6B JA voice model (#8207)
* Add SenseVoice speech-to-text model (Korean/Japanese support) SenseVoice (zh/en/ja/ko/yue) is the only bundled local STT model with Korean and Japanese support. The existing local models cover only English and Chinese (Parakeet, Zipformer, Paraformer); Whisper Tiny is multilingual but trades accuracy for breadth. - Add 'senseVoice' to SpeechModelType - Register the sherpa-onnx SenseVoice archive in the model catalog (pinned SHA-256, single-file model.int8.onnx + tokens.txt layout) - Handle the senseVoice type in the STT worker via createOfflineRecognizer with the senseVoice model config (auto language detection + ITN) - Add model-catalog regression tests for the new entry Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(speech): add Parakeet TDT-CTC 0.6B JA to the speech model catalog * test(speech): cover stt-worker-model-config file resolution incl. single-file models * feat(speech): decode Parakeet TDT-CTC JA via sherpa-onnx nemoCtc offline recognizer * fix(speech): use int8-only SenseVoice archive * fix(speech): refresh SenseVoice catalog metadata --------- Co-authored-by: xsacdw <xsacdw@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: LauraGPT <LauraGPT@users.noreply.github.com> Co-authored-by: Neil <4138956+nwparker@users.noreply.github.com>
This commit is contained in:
parent
ca8481ec73
commit
f5f026649e
|
|
@ -1,7 +1,33 @@
|
|||
import { describe, expect, it } from 'vitest'
|
||||
import { getCatalogModel } from './model-catalog'
|
||||
import { getCatalogModel, SPEECH_MODEL_CATALOG } from './model-catalog'
|
||||
|
||||
describe('SPEECH_MODEL_CATALOG', () => {
|
||||
it('includes the Japanese Parakeet TDT-CTC model with a valid manifest', () => {
|
||||
const manifest = getCatalogModel('parakeet-tdt-ctc-0.6b-ja-int8')
|
||||
|
||||
expect(manifest).toBeDefined()
|
||||
expect(manifest?.type).toBe('nemo-ctc')
|
||||
expect(manifest?.provider).toBe('local')
|
||||
expect(manifest?.language).toBe('ja')
|
||||
expect(manifest?.streaming).toBe(false)
|
||||
expect(manifest?.sampleRate).toBe(16000)
|
||||
expect(manifest?.files).toEqual(['model.int8.onnx', 'tokens.txt'])
|
||||
expect(manifest?.archiveFormat).toBe('tar.bz2')
|
||||
expect(manifest?.sizeBytes).toBe(489_389_564)
|
||||
expect(manifest?.downloadUrl).toBe(
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2'
|
||||
)
|
||||
expect(manifest?.archiveSha256).toBe(
|
||||
'4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b'
|
||||
)
|
||||
})
|
||||
|
||||
it('has unique ids across the catalog', () => {
|
||||
const ids = SPEECH_MODEL_CATALOG.map((m) => m.id)
|
||||
|
||||
expect(new Set(ids).size).toBe(ids.length)
|
||||
})
|
||||
|
||||
describe('SPEECH_MODEL_CATALOG SenseVoice entry', () => {
|
||||
it('registers SenseVoice as a non-streaming local model', () => {
|
||||
const model = getCatalogModel('sense-voice-zh-en-ja-ko-yue')
|
||||
expect(model).toBeDefined()
|
||||
|
|
|
|||
|
|
@ -145,6 +145,22 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
|
|||
streaming: true,
|
||||
modelingUnit: 'bpe'
|
||||
},
|
||||
{
|
||||
id: 'parakeet-tdt-ctc-0.6b-ja-int8',
|
||||
label: 'Parakeet TDT-CTC JA',
|
||||
description: 'Japanese only. Trained on 35k+ hours of natural speech. Punctuation included.',
|
||||
type: 'nemo-ctc',
|
||||
provider: 'local',
|
||||
language: 'ja',
|
||||
sizeBytes: 489_389_564,
|
||||
downloadUrl:
|
||||
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2',
|
||||
archiveSha256: '4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b',
|
||||
archiveFormat: 'tar.bz2',
|
||||
files: ['model.int8.onnx', 'tokens.txt'],
|
||||
sampleRate: 16000,
|
||||
streaming: false
|
||||
},
|
||||
{
|
||||
id: 'whisper-tiny',
|
||||
label: 'Whisper Tiny',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { resolveFile, resolveTokens } from './stt-worker-model-config'
|
||||
|
||||
const MODEL_DIR = join('models', 'test-model')
|
||||
|
||||
describe('resolveFile', () => {
|
||||
it('resolves an encoder/decoder/joiner triple by role name', () => {
|
||||
const files = ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt']
|
||||
|
||||
expect(resolveFile(files, 'encoder', MODEL_DIR)).toBe(join(MODEL_DIR, 'encoder.int8.onnx'))
|
||||
expect(resolveFile(files, 'decoder', MODEL_DIR)).toBe(join(MODEL_DIR, 'decoder.int8.onnx'))
|
||||
expect(resolveFile(files, 'joiner', MODEL_DIR)).toBe(join(MODEL_DIR, 'joiner.int8.onnx'))
|
||||
})
|
||||
|
||||
it('resolves a single fused model file for nemo-ctc-style manifests', () => {
|
||||
const files = ['model.int8.onnx', 'tokens.txt']
|
||||
|
||||
expect(resolveFile(files, 'model', MODEL_DIR)).toBe(join(MODEL_DIR, 'model.int8.onnx'))
|
||||
})
|
||||
|
||||
it('throws when no file matches the requested role', () => {
|
||||
const files = ['model.int8.onnx', 'tokens.txt']
|
||||
|
||||
expect(() => resolveFile(files, 'joiner', MODEL_DIR)).toThrow(/No \*joiner\*\.onnx found/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveTokens', () => {
|
||||
it('resolves tokens.txt regardless of surrounding files', () => {
|
||||
const files = ['model.int8.onnx', 'tokens.txt']
|
||||
|
||||
expect(resolveTokens(files, MODEL_DIR)).toBe(join(MODEL_DIR, 'tokens.txt'))
|
||||
})
|
||||
|
||||
it('throws when tokens.txt is missing', () => {
|
||||
const files = ['model.int8.onnx']
|
||||
|
||||
expect(() => resolveTokens(files, MODEL_DIR)).toThrow(/No \*tokens\.txt found/)
|
||||
})
|
||||
})
|
||||
|
|
@ -111,6 +111,22 @@ function handleInit(msg: Extract<WorkerMessage, { type: 'init' }>): void {
|
|||
}
|
||||
recognizer = sherpa.createOfflineRecognizer(config)
|
||||
stream = sherpa.createOfflineStream(recognizer)
|
||||
} else if (modelType === 'nemo-ctc') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
modelConfig: {
|
||||
nemoCtc: {
|
||||
model: resolveFile(files, 'model', modelDir)
|
||||
},
|
||||
tokens,
|
||||
numThreads: 2,
|
||||
provider: 'cpu',
|
||||
debug: 0
|
||||
},
|
||||
decodingMethod: 'greedy_search'
|
||||
}
|
||||
recognizer = sherpa.createOfflineRecognizer(config)
|
||||
stream = sherpa.createOfflineStream(recognizer)
|
||||
} else if (modelType === 'senseVoice') {
|
||||
const config = {
|
||||
featConfig: { sampleRate, featureDim: 80 },
|
||||
|
|
|
|||
|
|
@ -1,4 +1,10 @@
|
|||
export type SpeechModelType = 'transducer' | 'paraformer' | 'whisper' | 'senseVoice' | 'openai'
|
||||
export type SpeechModelType =
|
||||
| 'transducer'
|
||||
| 'paraformer'
|
||||
| 'whisper'
|
||||
| 'senseVoice'
|
||||
| 'nemo-ctc'
|
||||
| 'openai'
|
||||
export type SpeechModelProvider = 'local' | 'openai'
|
||||
|
||||
export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe'
|
||||
|
|
|
|||
Loading…
Reference in New Issue