feat(speech): add Parakeet TDT-CTC 0.6B JA voice model (#8207)

* Add SenseVoice speech-to-text model (Korean/Japanese support)

SenseVoice (zh/en/ja/ko/yue) is the only bundled local STT model with
Korean and Japanese support. The existing local models cover only
English and Chinese (Parakeet, Zipformer, Paraformer); Whisper Tiny is
multilingual but trades accuracy for breadth.

- Add 'senseVoice' to SpeechModelType
- Register the sherpa-onnx SenseVoice archive in the model catalog
  (pinned SHA-256, single-file model.int8.onnx + tokens.txt layout)
- Handle the senseVoice type in the STT worker via createOfflineRecognizer
  with the senseVoice model config (auto language detection + ITN)
- Add model-catalog regression tests for the new entry

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* feat(speech): add Parakeet TDT-CTC 0.6B JA to the speech model catalog

* test(speech): cover stt-worker-model-config file resolution incl. single-file models

* feat(speech): decode Parakeet TDT-CTC JA via sherpa-onnx nemoCtc offline recognizer

* fix(speech): use int8-only SenseVoice archive

* fix(speech): refresh SenseVoice catalog metadata

---------

Co-authored-by: xsacdw <xsacdw@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Co-authored-by: LauraGPT <LauraGPT@users.noreply.github.com>
Co-authored-by: Neil <4138956+nwparker@users.noreply.github.com>
This commit is contained in:
reopard007 2026-07-26 19:01:41 +09:00 committed by GitHub
parent ca8481ec73
commit f5f026649e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 108 additions and 3 deletions

View File

@ -1,7 +1,33 @@
import { describe, expect, it } from 'vitest'
import { getCatalogModel } from './model-catalog'
import { getCatalogModel, SPEECH_MODEL_CATALOG } from './model-catalog'
describe('SPEECH_MODEL_CATALOG', () => {
it('includes the Japanese Parakeet TDT-CTC model with a valid manifest', () => {
const manifest = getCatalogModel('parakeet-tdt-ctc-0.6b-ja-int8')
expect(manifest).toBeDefined()
expect(manifest?.type).toBe('nemo-ctc')
expect(manifest?.provider).toBe('local')
expect(manifest?.language).toBe('ja')
expect(manifest?.streaming).toBe(false)
expect(manifest?.sampleRate).toBe(16000)
expect(manifest?.files).toEqual(['model.int8.onnx', 'tokens.txt'])
expect(manifest?.archiveFormat).toBe('tar.bz2')
expect(manifest?.sizeBytes).toBe(489_389_564)
expect(manifest?.downloadUrl).toBe(
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2'
)
expect(manifest?.archiveSha256).toBe(
'4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b'
)
})
it('has unique ids across the catalog', () => {
const ids = SPEECH_MODEL_CATALOG.map((m) => m.id)
expect(new Set(ids).size).toBe(ids.length)
})
describe('SPEECH_MODEL_CATALOG SenseVoice entry', () => {
it('registers SenseVoice as a non-streaming local model', () => {
const model = getCatalogModel('sense-voice-zh-en-ja-ko-yue')
expect(model).toBeDefined()

View File

@ -145,6 +145,22 @@ export const SPEECH_MODEL_CATALOG: SpeechModelManifest[] = [
streaming: true,
modelingUnit: 'bpe'
},
{
id: 'parakeet-tdt-ctc-0.6b-ja-int8',
label: 'Parakeet TDT-CTC JA',
description: 'Japanese only. Trained on 35k+ hours of natural speech. Punctuation included.',
type: 'nemo-ctc',
provider: 'local',
language: 'ja',
sizeBytes: 489_389_564,
downloadUrl:
'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-parakeet-tdt_ctc-0.6b-ja-35000-int8.tar.bz2',
archiveSha256: '4b0a800ef29f4f4c8667339bf6f60d5bfdc2852ddc9dc5741aea65b6f8d1306b',
archiveFormat: 'tar.bz2',
files: ['model.int8.onnx', 'tokens.txt'],
sampleRate: 16000,
streaming: false
},
{
id: 'whisper-tiny',
label: 'Whisper Tiny',

View File

@ -0,0 +1,41 @@
import { join } from 'node:path'
import { describe, expect, it } from 'vitest'
import { resolveFile, resolveTokens } from './stt-worker-model-config'
const MODEL_DIR = join('models', 'test-model')
describe('resolveFile', () => {
it('resolves an encoder/decoder/joiner triple by role name', () => {
const files = ['encoder.int8.onnx', 'decoder.int8.onnx', 'joiner.int8.onnx', 'tokens.txt']
expect(resolveFile(files, 'encoder', MODEL_DIR)).toBe(join(MODEL_DIR, 'encoder.int8.onnx'))
expect(resolveFile(files, 'decoder', MODEL_DIR)).toBe(join(MODEL_DIR, 'decoder.int8.onnx'))
expect(resolveFile(files, 'joiner', MODEL_DIR)).toBe(join(MODEL_DIR, 'joiner.int8.onnx'))
})
it('resolves a single fused model file for nemo-ctc-style manifests', () => {
const files = ['model.int8.onnx', 'tokens.txt']
expect(resolveFile(files, 'model', MODEL_DIR)).toBe(join(MODEL_DIR, 'model.int8.onnx'))
})
it('throws when no file matches the requested role', () => {
const files = ['model.int8.onnx', 'tokens.txt']
expect(() => resolveFile(files, 'joiner', MODEL_DIR)).toThrow(/No \*joiner\*\.onnx found/)
})
})
describe('resolveTokens', () => {
it('resolves tokens.txt regardless of surrounding files', () => {
const files = ['model.int8.onnx', 'tokens.txt']
expect(resolveTokens(files, MODEL_DIR)).toBe(join(MODEL_DIR, 'tokens.txt'))
})
it('throws when tokens.txt is missing', () => {
const files = ['model.int8.onnx']
expect(() => resolveTokens(files, MODEL_DIR)).toThrow(/No \*tokens\.txt found/)
})
})

View File

@ -111,6 +111,22 @@ function handleInit(msg: Extract<WorkerMessage, { type: 'init' }>): void {
}
recognizer = sherpa.createOfflineRecognizer(config)
stream = sherpa.createOfflineStream(recognizer)
} else if (modelType === 'nemo-ctc') {
const config = {
featConfig: { sampleRate, featureDim: 80 },
modelConfig: {
nemoCtc: {
model: resolveFile(files, 'model', modelDir)
},
tokens,
numThreads: 2,
provider: 'cpu',
debug: 0
},
decodingMethod: 'greedy_search'
}
recognizer = sherpa.createOfflineRecognizer(config)
stream = sherpa.createOfflineStream(recognizer)
} else if (modelType === 'senseVoice') {
const config = {
featConfig: { sampleRate, featureDim: 80 },

View File

@ -1,4 +1,10 @@
export type SpeechModelType = 'transducer' | 'paraformer' | 'whisper' | 'senseVoice' | 'openai'
export type SpeechModelType =
| 'transducer'
| 'paraformer'
| 'whisper'
| 'senseVoice'
| 'nemo-ctc'
| 'openai'
export type SpeechModelProvider = 'local' | 'openai'
export type ModelingUnit = 'bpe' | 'cjkchar' | 'cjkchar+bpe'