From ad1cf02cd791b37f23092a8dcbb35484264ed288 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Fri, 10 Apr 2026 20:44:09 +0800 Subject: [PATCH] feat(mobile): add streaming tts playback --- apps/mobile/src/components/ui/form/Select.tsx | 3 + .../src/components/ui/grouped/GroupedList.tsx | 14 +- apps/mobile/src/lib/player.ts | 42 +- .../mobile/src/modules/context-menu/entry.tsx | 17 + .../EntryContentHeaderRightActions.tsx | 17 +- .../src/modules/player/GlassPlayerTabBar.tsx | 13 +- .../src/modules/player/PlayerTabBar.tsx | 13 +- .../src/modules/player/TtsStreamProvider.tsx | 53 +++ apps/mobile/src/modules/player/control.tsx | 62 ++- apps/mobile/src/modules/player/entry-tts.ts | 107 +++++ apps/mobile/src/modules/player/tts-core.ts | 115 ++++++ .../src/modules/player/tts-service.test.ts | 61 +++ apps/mobile/src/modules/player/tts-service.ts | 78 ++++ .../modules/player/tts-stream-controller.ts | 288 +++++++++++++ .../modules/player/tts-stream-webview-html.ts | 377 ++++++++++++++++++ .../src/modules/settings/routes/General.tsx | 47 +++ apps/mobile/src/providers/index.tsx | 2 + apps/mobile/src/screens/PlayerScreen.tsx | 22 +- locales/mobile/default/en.json | 2 + locales/mobile/default/fr-FR.json | 2 + locales/mobile/default/ja.json | 2 + locales/mobile/default/zh-CN.json | 2 + locales/mobile/default/zh-TW.json | 2 + locales/settings/en.json | 1 + locales/settings/fr-FR.json | 1 + locales/settings/ja.json | 1 + locales/settings/zh-CN.json | 1 + locales/settings/zh-TW.json | 1 + 28 files changed, 1317 insertions(+), 29 deletions(-) create mode 100644 apps/mobile/src/modules/player/TtsStreamProvider.tsx create mode 100644 apps/mobile/src/modules/player/entry-tts.ts create mode 100644 apps/mobile/src/modules/player/tts-core.ts create mode 100644 apps/mobile/src/modules/player/tts-service.test.ts create mode 100644 apps/mobile/src/modules/player/tts-service.ts create mode 100644 apps/mobile/src/modules/player/tts-stream-controller.ts create mode 100644 apps/mobile/src/modules/player/tts-stream-webview-html.ts diff --git a/apps/mobile/src/components/ui/form/Select.tsx b/apps/mobile/src/components/ui/form/Select.tsx index 321212093..8b23902d8 100644 --- a/apps/mobile/src/components/ui/form/Select.tsx +++ b/apps/mobile/src/components/ui/form/Select.tsx @@ -23,6 +23,7 @@ interface SelectProps { wrapperStyle?: StyleProp label?: string disabled?: boolean + triggerTestID?: string } export function Select({ options, @@ -33,6 +34,7 @@ export function Select({ wrapperStyle, label, disabled, + triggerTestID, }: SelectProps) { const [currentValue, setCurrentValue] = useState(() => value) useEffect(() => { @@ -49,6 +51,7 @@ export function Select({ void + testID?: string } & BaseCellClassNames -> = ({ label, description, children, className, leftClassName, rightClassName, icon, onPress }) => { +> = ({ + label, + description, + children, + className, + leftClassName, + rightClassName, + icon, + onPress, + testID, +}) => { return ( )} + { + void playEntryTts(id, { + toastTitle: t("entry_content.header.play_tts"), + }) + }} + > + + {t("entry_content.header.play_tts")} + + {entry.url && ( , + iconIOS: { name: "speaker.wave.2" }, + onPress: () => { + void playEntryTts(entryId, { + preferReadability: showReadability, + toastTitle: t("entry_content.header.play_tts"), + }) + }, + inMenu: true, + }, { key: "Share", title: t("operation.share"), @@ -228,7 +243,7 @@ const HeaderRightActionsImpl = ({ - + diff --git a/apps/mobile/src/modules/player/GlassPlayerTabBar.tsx b/apps/mobile/src/modules/player/GlassPlayerTabBar.tsx index 22ddf9227..704ca0fc6 100644 --- a/apps/mobile/src/modules/player/GlassPlayerTabBar.tsx +++ b/apps/mobile/src/modules/player/GlassPlayerTabBar.tsx @@ -8,7 +8,7 @@ import { Image } from "@/src/components/ui/image/Image" import { Text } from "@/src/components/ui/typography/Text" import { BottomTabContext } from "@/src/lib/navigation/bottom-tab/BottomTabContext" import { useNavigation } from "@/src/lib/navigation/hooks" -import { useActiveTrack } from "@/src/lib/player" +import { useActivePlayable } from "@/src/lib/player" import { PlayerScreen } from "@/src/screens/PlayerScreen" import { usePrefetchImageColors } from "@/src/store/image/hooks" @@ -16,15 +16,15 @@ import { PlayPauseButton, SeekButton } from "./control" const allowedTabIdentifiers = new Set(["IndexTabScreen", "SubscriptionsTabScreen"]) export function GlassPlayerTabBar({ className }: { className?: string }) { - const activeTrack = useActiveTrack() + const activePlayable = useActivePlayable() const tabRootCtx = use(BottomTabContext) const tabScreens = useAtomValue(tabRootCtx.tabScreensAtom) const currentIndex = useAtomValue(tabRootCtx.currentIndexAtom) const currentTabProps = tabScreens.find((tabScreen) => tabScreen.tabScreenIndex === currentIndex) const identifier = currentTabProps?.identifier - const isVisible = !!activeTrack && identifier && allowedTabIdentifiers.has(identifier) + const isVisible = !!activePlayable && identifier && allowedTabIdentifiers.has(identifier) - usePrefetchImageColors(activeTrack?.artwork) + usePrefetchImageColors(activePlayable?.artwork ?? undefined) const navigation = useNavigation() if (!isVisible) return null @@ -34,6 +34,7 @@ export function GlassPlayerTabBar({ className }: { className?: string }) { { navigation.presentControllerView(PlayerScreen, void 0, "transparentModal") }} @@ -41,13 +42,13 @@ export function GlassPlayerTabBar({ className }: { className?: string }) { - {activeTrack?.title ?? ""} + {activePlayable?.title ?? ""} diff --git a/apps/mobile/src/modules/player/PlayerTabBar.tsx b/apps/mobile/src/modules/player/PlayerTabBar.tsx index 7dd0bdaea..2d54a9ba3 100644 --- a/apps/mobile/src/modules/player/PlayerTabBar.tsx +++ b/apps/mobile/src/modules/player/PlayerTabBar.tsx @@ -13,7 +13,7 @@ import { Image } from "@/src/components/ui/image/Image" import { Text } from "@/src/components/ui/typography/Text" import { BottomTabContext } from "@/src/lib/navigation/bottom-tab/BottomTabContext" import { useNavigation } from "@/src/lib/navigation/hooks" -import { useActiveTrack } from "@/src/lib/player" +import { useActivePlayable } from "@/src/lib/player" import { PlayerScreen } from "@/src/screens/PlayerScreen" import { usePrefetchImageColors } from "@/src/store/image/hooks" @@ -21,13 +21,13 @@ import { PlayPauseButton, SeekButton } from "./control" const allowedTabIdentifiers = new Set(["IndexTabScreen", "SubscriptionsTabScreen"]) export function PlayerTabBar({ className }: { className?: string }) { - const activeTrack = useActiveTrack() + const activePlayable = useActivePlayable() const tabRootCtx = use(BottomTabContext) const tabScreens = useAtomValue(tabRootCtx.tabScreensAtom) const currentIndex = useAtomValue(tabRootCtx.currentIndexAtom) const currentTabProps = tabScreens.find((tabScreen) => tabScreen.tabScreenIndex === currentIndex) const identifier = currentTabProps?.identifier - const isVisible = !!activeTrack && identifier && allowedTabIdentifiers.has(identifier) + const isVisible = !!activePlayable && identifier && allowedTabIdentifiers.has(identifier) const isVisibleSV = useSharedValue(isVisible ? 1 : 0) useEffect(() => { isVisibleSV.value = withTiming(isVisible ? 1 : 0) @@ -39,7 +39,7 @@ export function PlayerTabBar({ className }: { className?: string }) { overflow: "hidden", } }) - usePrefetchImageColors(activeTrack?.artwork) + usePrefetchImageColors(activePlayable?.artwork ?? undefined) const navigation = useNavigation() return ( { navigation.presentControllerView(PlayerScreen, void 0, "transparentModal") }} @@ -54,13 +55,13 @@ export function PlayerTabBar({ className }: { className?: string }) { - {activeTrack?.title ?? ""} + {activePlayable?.title ?? ""} diff --git a/apps/mobile/src/modules/player/TtsStreamProvider.tsx b/apps/mobile/src/modules/player/TtsStreamProvider.tsx new file mode 100644 index 000000000..d4e138ac6 --- /dev/null +++ b/apps/mobile/src/modules/player/TtsStreamProvider.tsx @@ -0,0 +1,53 @@ +import { useEffect, useRef } from "react" +import { StyleSheet, View } from "react-native" +import { WebView } from "react-native-webview" + +import { ttsStreamController } from "./tts-stream-controller" +import { TTS_STREAM_WEBVIEW_HTML } from "./tts-stream-webview-html" + +export const TtsStreamProvider = () => { + const webViewRef = useRef(null) + + useEffect(() => { + ttsStreamController.attachWebView(webViewRef.current) + + return () => { + ttsStreamController.attachWebView(null) + } + }, []) + + return ( + + + + ) +} + +const styles = StyleSheet.create({ + container: { + height: 1, + left: -10_000, + opacity: 0, + pointerEvents: "none", + position: "absolute", + top: 0, + width: 1, + }, + webView: { + backgroundColor: "transparent", + height: 1, + width: 1, + }, +}) diff --git a/apps/mobile/src/modules/player/control.tsx b/apps/mobile/src/modules/player/control.tsx index 0d9b3cf81..00aae6d99 100644 --- a/apps/mobile/src/modules/player/control.tsx +++ b/apps/mobile/src/modules/player/control.tsx @@ -16,11 +16,19 @@ import { StopCircleCuteFiIcon } from "@/src/icons/stop_circle_cute_fi" import { VolumeCuteReIcon } from "@/src/icons/volume_cute_re" import { VolumeOffCuteReIcon } from "@/src/icons/volume_off_cute_re" import { useNavigation } from "@/src/lib/navigation/hooks" -import { allowedRate, player, useIsPlaying, useProgress, useRate } from "@/src/lib/player" +import { + allowedRate, + player, + useIsPlaying, + useProgress, + useRate, + useTtsStreamPlayback, +} from "@/src/lib/player" import { useVolume } from "@/src/lib/volume" import { useColor } from "@/src/theme/colors" import { usePlayerScreenContext } from "./context" +import { ttsStreamController } from "./tts-stream-controller" type ControlButtonProps = { size?: number @@ -28,19 +36,33 @@ type ControlButtonProps = { color?: string } export function PlayPauseButton({ size = 24, className, color }: ControlButtonProps) { + const ttsStream = useTtsStreamPlayback() const { playing } = useIsPlaying() + const isStreamPlaying = ttsStream.status === "playing" + const isStream = !!ttsStream.entryId const label = useColor("label") return ( { + if (isStream && ttsStream.entryId) { + void ttsStreamController.toggle(ttsStream.entryId) + return + } + playing ? player.pause() : player.play() }} > - {playing ? ( + {isStream ? ( + isStreamPlaying ? ( + + ) : ( + + ) + ) : playing ? ( ) : ( @@ -58,6 +80,10 @@ export function SeekButton({ offset?: number }) { const label = useColor("label") + const ttsStream = useTtsStreamPlayback() + if (ttsStream.entryId) { + return null + } return ( @@ -110,13 +140,18 @@ export function RateSelector() { ) } export function StopButton({ size = 24, className, color }: ControlButtonProps) { + const ttsStream = useTtsStreamPlayback() const label = useColor("label") const navigation = useNavigation() return ( { - player.reset() + if (ttsStream.entryId) { + void ttsStreamController.stop() + } else { + player.reset() + } navigation.back() }} > @@ -125,8 +160,19 @@ export function StopButton({ size = 24, className, color }: ControlButtonProps) ) } export function ControlGroup() { + const ttsStream = useTtsStreamPlayback() const { isBackgroundLight } = usePlayerScreenContext() const buttonColor = isBackgroundLight ? "black" : "white" + + if (ttsStream.entryId) { + return ( + + + + + ) + } + return ( @@ -155,6 +201,10 @@ export function ProgressBar() { }) const min = useSharedValue(0) const max = useSharedValue(1) + const ttsStream = useTtsStreamPlayback() + if (ttsStream.entryId) { + return null + } const trackElapsedTime = formatSecondsToMinutes(position) const trackRemainingTime = formatSecondsToMinutes(duration - position) return ( @@ -213,6 +263,10 @@ export function VolumeBar() { const progress = useSharedValue(0) const min = useSharedValue(0) const max = useSharedValue(1) + const ttsStream = useTtsStreamPlayback() + if (ttsStream.entryId) { + return null + } progress.value = volume ?? 0 return ( diff --git a/apps/mobile/src/modules/player/entry-tts.ts b/apps/mobile/src/modules/player/entry-tts.ts new file mode 100644 index 000000000..5c9d30a8e --- /dev/null +++ b/apps/mobile/src/modules/player/entry-tts.ts @@ -0,0 +1,107 @@ +import { getEntry } from "@follow/store/entry/getter" +import { getFeedById } from "@follow/store/feed/getter" +import TrackPlayer, { State } from "react-native-track-player" + +import { getGeneralSettings } from "@/src/atoms/settings/general" +import { toastFetchError } from "@/src/lib/error-parser" +import { player } from "@/src/lib/player" + +import { getEntryTtsText, requestTtsFile } from "./tts-service" +import { ttsStreamController } from "./tts-stream-controller" + +let activeTtsEntryId: string | null = null +let activeTtsTrackUrl: string | null = null + +const isSameEntryTtsTrack = async (entryId: string) => { + if (ttsStreamController.canToggleEntry(entryId)) { + return true + } + + if (!activeTtsEntryId || !activeTtsTrackUrl || activeTtsEntryId !== entryId) { + return false + } + + const activeTrack = await TrackPlayer.getActiveTrack() + return activeTrack?.url === activeTtsTrackUrl +} + +const toggleCurrentTtsPlayback = async () => { + if (activeTtsEntryId && ttsStreamController.canToggleEntry(activeTtsEntryId)) { + await ttsStreamController.toggle(activeTtsEntryId) + return + } + + const { state } = await TrackPlayer.getPlaybackState() + + if ([State.Playing, State.Buffering, State.Loading].includes(state)) { + await TrackPlayer.pause() + return + } + + await TrackPlayer.play() +} + +export const playEntryTts = async ( + entryId: string, + { + preferReadability = false, + toastTitle, + }: { + preferReadability?: boolean + toastTitle: string + }, +) => { + try { + if (await isSameEntryTtsTrack(entryId)) { + await toggleCurrentTtsPlayback() + return + } + + const entry = getEntry(entryId) + if (!entry) { + throw new Error("Entry not found") + } + + const text = getEntryTtsText(entry, { preferReadability }) + if (!text) { + throw new Error("No content available for TTS") + } + + const { voice } = getGeneralSettings() + const feed = getFeedById(entry.feedId) + try { + await ttsStreamController.play({ + artwork: entry.media?.find((media) => media.type === "photo")?.url ?? null, + artist: feed?.title ?? "Folo", + entryId, + text, + title: entry.title || toastTitle, + voice, + }) + + activeTtsEntryId = entryId + activeTtsTrackUrl = null + return + } catch { + // Fall back to the buffered native player when the streaming WebView is not ready. + } + + const trackUrl = await requestTtsFile({ + cacheKey: entryId, + text, + voice, + }) + + await player.play({ + artwork: entry.media?.find((media) => media.type === "photo")?.url ?? undefined, + artist: feed?.title ?? "Folo", + title: entry.title || toastTitle, + url: trackUrl, + }) + + activeTtsEntryId = entryId + activeTtsTrackUrl = trackUrl + } catch (error) { + toastFetchError(error as Error, { title: toastTitle }) + } +} diff --git a/apps/mobile/src/modules/player/tts-core.ts b/apps/mobile/src/modules/player/tts-core.ts new file mode 100644 index 000000000..ca086cfe0 --- /dev/null +++ b/apps/mobile/src/modules/player/tts-core.ts @@ -0,0 +1,115 @@ +import type { EntryModel } from "@follow/store/entry/types" +import { parseHtml } from "@follow/utils/html" + +export const TTS_SERVICE_URL = "https://tts.folo.is" +export const DEFAULT_TTS_VOICE = "en-US-AvaMultilingualNeural" + +export interface TtsVoice { + FriendlyName: string + Gender: string + Locale: string + ShortName: string +} + +interface TtsVoiceResponse { + voices: TtsVoice[] +} + +interface TtsErrorResponse { + error?: { + message?: string + } +} + +const normalizeTtsText = (value: string) => + value + .replaceAll("\r\n", "\n") + .replaceAll(/[^\S\n]+/g, " ") + .replaceAll(/\n{3,}/g, "\n\n") + .trim() + +const toPlainText = (value: string) => normalizeTtsText(parseHtml(value).toText()) + +const readTtsErrorMessage = async (response: Response) => { + try { + const data = (await response.clone().json()) as TtsErrorResponse + return data?.error?.message || "TTS request failed" + } catch { + return "TTS request failed" + } +} + +export const getEntryTtsText = ( + entry: Pick, + options?: { + preferReadability?: boolean + }, +) => { + const { preferReadability = false } = options ?? {} + + const title = normalizeTtsText(entry.title || "") + const bodySource = preferReadability + ? entry.readabilityContent || entry.content || entry.description || "" + : entry.content || entry.description || entry.readabilityContent || "" + const body = bodySource ? toPlainText(bodySource) : "" + + return [title, body].filter(Boolean).join("\n\n") +} + +export const fetchTtsVoices = async ({ + fetch, + signal, +}: { + fetch: typeof globalThis.fetch + signal?: AbortSignal +}) => { + const response = await fetch(`${TTS_SERVICE_URL}/voices`, { signal }) + + if (!response.ok) { + throw new Error(await readTtsErrorMessage(response)) + } + + const data = (await response.json()) as TtsVoiceResponse + return data.voices ?? [] +} + +export const requestTtsBytes = async ({ + fetch, + signal, + text, + voice, +}: { + fetch: typeof globalThis.fetch + signal?: AbortSignal + text: string + voice?: string +}) => { + const normalizedText = normalizeTtsText(text) + if (!normalizedText) { + throw new Error("Text is required") + } + + const normalizedVoice = voice?.trim() + + const response = await fetch(`${TTS_SERVICE_URL}/tts`, { + body: JSON.stringify({ + text: normalizedText, + ...(normalizedVoice ? { voice: normalizedVoice } : {}), + }), + headers: { + "Content-Type": "application/json", + }, + method: "POST", + signal, + }) + + if (!response.ok) { + throw new Error(await readTtsErrorMessage(response)) + } + + if ("bytes" in response && typeof response.bytes === "function") { + return response.bytes() + } + + return new Uint8Array(await response.arrayBuffer()) +} diff --git a/apps/mobile/src/modules/player/tts-service.test.ts b/apps/mobile/src/modules/player/tts-service.test.ts new file mode 100644 index 000000000..651dce73c --- /dev/null +++ b/apps/mobile/src/modules/player/tts-service.test.ts @@ -0,0 +1,61 @@ +import assert from "node:assert/strict" +import { describe, it } from "node:test" + +describe("mobile tts service", () => { + it("extracts normalized plain text from entry content", async () => { + const { getEntryTtsText } = await import("./tts-core") + + assert.equal( + getEntryTtsText({ + title: " Hello world ", + content: "

Line 1

Line 2

", + description: "", + readabilityContent: "", + }), + "Hello world\n\nLine 1\n\nLine 2", + ) + }) + + it("posts the normalized text to the TTS service and writes the returned bytes to cache", async () => { + const calls: { + create?: unknown + request?: { + body?: string + headers?: Record + method?: string + } + written?: Uint8Array + } = {} + + const { requestTtsBytes } = await import("./tts-core") + + const bytes = await requestTtsBytes({ + fetch: async (_input, init) => { + calls.request = { + body: typeof init?.body === "string" ? init.body : undefined, + headers: init?.headers as Record, + method: init?.method, + } + + return { + ok: true, + bytes: async () => new Uint8Array([1, 2, 3]), + } as Response & { bytes: () => Promise } + }, + text: " Hello world ", + voice: "en-US-AvaMultilingualNeural", + }) + + assert.deepEqual(calls.request, { + body: JSON.stringify({ + text: "Hello world", + voice: "en-US-AvaMultilingualNeural", + }), + headers: { + "Content-Type": "application/json", + }, + method: "POST", + }) + assert.deepEqual(bytes, new Uint8Array([1, 2, 3])) + }) +}) diff --git a/apps/mobile/src/modules/player/tts-service.ts b/apps/mobile/src/modules/player/tts-service.ts new file mode 100644 index 000000000..333364302 --- /dev/null +++ b/apps/mobile/src/modules/player/tts-service.ts @@ -0,0 +1,78 @@ +import { fetch as expoFetch } from "expo/fetch" +import { Directory, File, Paths } from "expo-file-system" + +import { fetchTtsVoices as fetchTtsVoicesCore, requestTtsBytes } from "./tts-core" + +export { DEFAULT_TTS_VOICE, getEntryTtsText, TTS_SERVICE_URL, type TtsVoice } from "./tts-core" + +interface TtsCacheFile { + uri: string + write: (content: Uint8Array) => void +} + +interface TtsDependencies { + createCacheFile: (cacheKey: string) => TtsCacheFile + fetch: typeof expoFetch +} + +const sanitizeCacheKey = (value: string) => + value + .trim() + .replaceAll(/[^\w-]+/g, "_") + .replaceAll(/^_+|_+$/g, "") || "tts" + +const createCacheFile = (cacheKey: string): TtsCacheFile => { + const directory = new Directory(Paths.cache, "tts") + directory.create({ + idempotent: true, + intermediates: true, + }) + + const file = new File(directory, `${sanitizeCacheKey(cacheKey)}-${Date.now()}.mp3`) + file.create({ + intermediates: true, + overwrite: true, + }) + + return { + uri: file.uri, + write(content) { + file.write(content) + }, + } +} + +const defaultDependencies: TtsDependencies = { + createCacheFile, + fetch: expoFetch, +} + +export const fetchTtsVoices = async ( + signal?: AbortSignal, + dependencies: Pick = defaultDependencies, +) => fetchTtsVoicesCore({ fetch: dependencies.fetch as typeof globalThis.fetch, signal }) + +export const requestTtsFile = async ({ + cacheKey, + dependencies = defaultDependencies, + signal, + text, + voice, +}: { + cacheKey: string + dependencies?: TtsDependencies + signal?: AbortSignal + text: string + voice?: string +}) => { + const file = dependencies.createCacheFile(cacheKey) + file.write( + await requestTtsBytes({ + fetch: dependencies.fetch as typeof globalThis.fetch, + signal, + text, + voice, + }), + ) + return file.uri +} diff --git a/apps/mobile/src/modules/player/tts-stream-controller.ts b/apps/mobile/src/modules/player/tts-stream-controller.ts new file mode 100644 index 000000000..74fb83553 --- /dev/null +++ b/apps/mobile/src/modules/player/tts-stream-controller.ts @@ -0,0 +1,288 @@ +import type { WebView as WebViewType, WebViewMessageEvent } from "react-native-webview" + +export type TtsPlaybackStatus = "idle" | "loading" | "paused" | "playing" + +export interface TtsStreamPlaybackState { + artwork?: string | null + artist?: string | null + entryId: string | null + status: TtsPlaybackStatus + title?: string | null +} + +type TtsBridgeCommand = + | { + entryId: string + requestId: string + text: string + type: "play" + voice?: string + } + | { + entryId: string + type: "toggle" + } + | { + type: "stop" + } + +type TtsBridgeEvent = + | { type: "ready" } + | { entryId: string; requestId: string; type: "started" } + | { entryId: string; type: "playing" | "paused" | "ended" } + | { entryId?: string; message: string; requestId?: string; type: "error" } + +const START_TIMEOUT_MS = 10_000 + +class TtsStreamController { + private listeners = new Set<() => void>() + + private playbackState: TtsStreamPlaybackState = { + artwork: null, + artist: null, + entryId: null, + status: "idle", + title: null, + } + + private pendingStart: { + reject: (reason?: unknown) => void + requestId: string + resolve: () => void + timeoutId: ReturnType + } | null = null + + private queuedCommands: string[] = [] + private ready = false + private readyWaiters = new Set<() => void>() + private webView: WebViewType | null = null + + attachWebView = (webView: WebViewType | null) => { + this.webView = webView + if (!webView) { + this.ready = false + } + } + + getState = () => this.playbackState + + canToggleEntry = (entryId: string) => + this.playbackState.entryId === entryId && this.playbackState.status !== "loading" + + subscribe = (listener: () => void) => { + this.listeners.add(listener) + return () => { + this.listeners.delete(listener) + } + } + + handleMessage = (event: WebViewMessageEvent) => { + let payload: TtsBridgeEvent + try { + payload = JSON.parse(event.nativeEvent.data) as TtsBridgeEvent + } catch { + return + } + + switch (payload.type) { + case "ready": { + this.ready = true + for (const resolve of this.readyWaiters) { + resolve() + } + this.readyWaiters.clear() + this.flushQueuedCommands() + return + } + case "started": { + this.setPlaybackState({ + status: "playing", + }) + this.resolvePendingStart(payload.requestId) + return + } + case "playing": + case "paused": { + this.setPlaybackState({ + status: payload.type, + }) + return + } + case "ended": { + if (this.playbackState.entryId === payload.entryId) { + this.resetPlaybackState() + } + return + } + case "error": { + this.rejectPendingStart(payload.requestId, new Error(payload.message)) + if (this.playbackState.entryId === payload.entryId) { + this.resetPlaybackState() + } + } + } + } + + play = async ({ + artwork, + artist, + entryId, + text, + title, + voice, + }: { + artwork?: string | null + artist?: string | null + entryId: string + text: string + title?: string | null + voice?: string + }) => { + await this.waitUntilReady() + + const requestId = `${entryId}-${Date.now()}` + + this.rejectPendingStart(undefined, new Error("TTS interrupted")) + this.setPlaybackState({ + artwork, + artist, + entryId, + status: "loading", + title, + }) + + await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + if (this.pendingStart?.requestId === requestId) { + this.pendingStart = null + this.resetPlaybackState() + reject(new Error("TTS streaming did not start in time")) + } + }, START_TIMEOUT_MS) + + this.pendingStart = { + reject, + requestId, + resolve, + timeoutId, + } + + this.sendCommand({ + entryId, + requestId, + text, + type: "play", + voice, + }) + }) + } + + toggle = async (entryId: string) => { + await this.waitUntilReady() + this.sendCommand({ + entryId, + type: "toggle", + }) + } + + stop = async () => { + this.rejectPendingStart(undefined, new Error("TTS interrupted")) + this.sendCommand({ + type: "stop", + }) + this.resetPlaybackState() + } + + private flushQueuedCommands = () => { + if (!this.webView || !this.ready) { + return + } + + for (const command of this.queuedCommands) { + this.webView.postMessage(command) + } + this.queuedCommands = [] + } + + private resolvePendingStart = (requestId?: string) => { + if (!this.pendingStart || this.pendingStart.requestId !== requestId) { + return + } + + clearTimeout(this.pendingStart.timeoutId) + this.pendingStart.resolve() + this.pendingStart = null + } + + private rejectPendingStart = (requestId?: string, error?: Error) => { + if (!this.pendingStart) { + return + } + + if (requestId && this.pendingStart.requestId !== requestId) { + return + } + + clearTimeout(this.pendingStart.timeoutId) + this.pendingStart.reject(error) + this.pendingStart = null + } + + private notify = () => { + for (const listener of this.listeners) { + listener() + } + } + + private resetPlaybackState = () => { + this.playbackState = { + artwork: null, + artist: null, + entryId: null, + status: "idle", + title: null, + } + this.notify() + } + + private setPlaybackState = (patch: Partial) => { + this.playbackState = { + ...this.playbackState, + ...patch, + } + this.notify() + } + + private sendCommand = (command: TtsBridgeCommand) => { + const serialized = JSON.stringify(command) + + if (!this.webView || !this.ready) { + this.queuedCommands.push(serialized) + return + } + + this.webView.postMessage(serialized) + } + + private waitUntilReady = () => { + if (this.ready && this.webView) { + return Promise.resolve() + } + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.readyWaiters.delete(handleReady) + reject(new Error("TTS streaming player is not ready")) + }, 5_000) + + const handleReady = () => { + clearTimeout(timeoutId) + resolve() + } + + this.readyWaiters.add(handleReady) + }) + } +} + +export const ttsStreamController = new TtsStreamController() diff --git a/apps/mobile/src/modules/player/tts-stream-webview-html.ts b/apps/mobile/src/modules/player/tts-stream-webview-html.ts new file mode 100644 index 000000000..2d9402c30 --- /dev/null +++ b/apps/mobile/src/modules/player/tts-stream-webview-html.ts @@ -0,0 +1,377 @@ +export const TTS_STREAM_WEBVIEW_HTML = String.raw` + + + + + + TTS Stream + + + + + +` diff --git a/apps/mobile/src/modules/settings/routes/General.tsx b/apps/mobile/src/modules/settings/routes/General.tsx index b2aa0e5e4..52106ec56 100644 --- a/apps/mobile/src/modules/settings/routes/General.tsx +++ b/apps/mobile/src/modules/settings/routes/General.tsx @@ -1,4 +1,5 @@ import { ACTION_LANGUAGE_KEYS } from "@follow/shared" +import { useQuery } from "@tanstack/react-query" import i18next from "i18next" import { useMemo } from "react" import { useTranslation } from "react-i18next" @@ -20,6 +21,7 @@ import { import { Switch } from "@/src/components/ui/switch/Switch" import { updateDayjsLocale } from "@/src/lib/i18n" import type { NavigationControllerView } from "@/src/lib/navigation/types" +import { fetchTtsVoices } from "@/src/modules/player/tts-service" const settingSelectWrapperClassName = "w-[200px]" @@ -112,6 +114,46 @@ function TranslationModeSetting() { ) } +function VoiceSetting() { + const { t } = useTranslation("settings") + const voice = useGeneralSettingKey("voice") + const { data } = useQuery({ + queryFn: ({ signal }) => fetchTtsVoices(signal), + queryKey: ["tts-voices"], + staleTime: Number.POSITIVE_INFINITY, + }) + + const options = useMemo( + () => + data?.map((item) => ({ + label: item.ShortName, + subLabel: item.FriendlyName, + value: item.ShortName, + })) ?? [{ label: voice, value: voice }], + [data, voice], + ) + + const selectedVoice = data?.find((item) => item.ShortName === voice) + + return ( + +