From e7409a4e7ab56d2ebf03e2df72ffa73506aca432 Mon Sep 17 00:00:00 2001 From: Caspian Date: Fri, 1 Dec 2023 17:53:49 +0000 Subject: [PATCH] feat: Invoke translateNote via API. --- src/app/[locale]/site/[site]/[slug]/page.tsx | 6 +- src/app/api/feed/route.ts | 6 +- src/app/api/translate-note/route.ts | 210 +++++++++++++++++++ src/components/home/HomeFeed.tsx | 11 +- src/lib/expand-unit.ts | 53 ++++- src/models/home.model.ts | 15 +- src/models/page.model.ts | 3 + src/queries/page.server.ts | 210 +------------------ 8 files changed, 285 insertions(+), 229 deletions(-) create mode 100644 src/app/api/translate-note/route.ts diff --git a/src/app/[locale]/site/[site]/[slug]/page.tsx b/src/app/[locale]/site/[site]/[slug]/page.tsx index 62f80a8a..6d20f044 100644 --- a/src/app/[locale]/site/[site]/[slug]/page.tsx +++ b/src/app/[locale]/site/[site]/[slug]/page.tsx @@ -16,6 +16,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser" import { isInRN } from "~/lib/is-in-rn" import { isOnlyContent } from "~/lib/is-only-content" import getQueryClient from "~/lib/query-client" +import { Language } from "~/lib/types" import { fetchGetPage, getSummary } from "~/queries/page.server" import { fetchGetSite } from "~/queries/site.server" @@ -28,7 +29,7 @@ export async function generateMetadata({ } }): Promise { const queryClient = getQueryClient() - + const locale = (await getLocale()) as Language const site = await fetchGetSite(params.site, queryClient) const page = await fetchGetPage( @@ -36,12 +37,11 @@ export async function generateMetadata({ characterId: site?.characterId, slug: params.slug, useStat: true, + translateTo: locale, }, queryClient, ) - // await decoratePageWithTranslation(page) - const title = `${ page?.metadata?.content?.title || page?.metadata?.content?.content } - ${site?.metadata?.content?.name || site?.handle}` diff --git a/src/app/api/feed/route.ts b/src/app/api/feed/route.ts index 39542c15..fe1bd4b2 100644 --- a/src/app/api/feed/route.ts +++ b/src/app/api/feed/route.ts @@ -1,6 +1,5 @@ import { getQuery, NextServerResponse } from "~/lib/server-helper" import { getFeed } from "~/models/home.model" -import { decoratePageWithTranslation } from "~/queries/page.server" export async function GET(req: Request) { const query = getQuery(req) @@ -16,12 +15,9 @@ export async function GET(req: Request) { tag: query.tag, useHTML: false, topic: query.topic, + translateTo: query.translateTo, }) - for (const item of result.list) { - await decoratePageWithTranslation(item) - } - const res = new NextServerResponse() return res.status(200).json(result) } diff --git a/src/app/api/translate-note/route.ts b/src/app/api/translate-note/route.ts new file mode 100644 index 00000000..586a89af --- /dev/null +++ b/src/app/api/translate-note/route.ts @@ -0,0 +1,210 @@ +import AsyncLock from "async-lock" +import { LLMChain } from "langchain/chains" +import { OpenAI } from "langchain/llms/openai" +import { PromptTemplate } from "langchain/prompts" + +import { Metadata } from "@prisma/client" + +import { languageNames } from "~/i18n" +import { toGateway } from "~/lib/ipfs-parser" +import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length" +import prisma from "~/lib/prisma.server" +import { cacheGet } from "~/lib/redis.server" +import { getQuery, NextServerResponse } from "~/lib/server-helper" +import { Language } from "~/lib/types" + +const lock = new AsyncLock() + +type ContentTranslation = { + title?: string + content?: string +} + +let translationModel4K: OpenAI | undefined +let translationModel16K: OpenAI | undefined +if (process.env.OPENAI_API_KEY) { + const options = { + openAIApiKey: process.env.OPENAI_API_KEY, + temperature: 0.2, + maxTokens: -1, + } + translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" }) + translationModel16K = new OpenAI({ + ...options, + modelName: "gpt-3.5-turbo-16k", + }) +} + +type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja" +const translationChains = new Map() + +const getOriginalTranslation = async ( + cid: string, + targetLang: Language, + originalLang?: Language, +): Promise => { + if (!translationModel4K || !translationModel16K) return + + try { + const { title, content } = await ( + await fetch(toGateway(`ipfs://${cid}`)) + ).json() + + // If the detected language is the same as the target language, return the original content + if (originalLang === targetLang) { + console.warn( + `|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`, + ) + return + } + + console.time(`fetching translation ${cid}, ${targetLang}`) + const { modelSize, tokens } = llmModelSwitcherByTextLength(content, { + includeResponse: { lang: targetLang }, + }) + + if (!modelSize) { + console.error( + `|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`, + ) + return + } + + let chain = translationChains.get(`${modelSize}_${targetLang}`) + + if (!chain) { + const prompt = new PromptTemplate({ + template: `Translate the following text into "${languageNames[targetLang]}" language: + {text} + Translation:`, + inputVariables: ["text"], + }) + + const translateModel = + modelSize === "4k" ? translationModel4K : translationModel16K + + chain = new LLMChain({ llm: translateModel, prompt }) + + translationChains.set(`${modelSize}_${targetLang}`, chain) + } + + const t = await chain.call({ text: title }) + const c = await chain.call({ text: content }) + + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + + return { + title: t.text, + content: c.text, + } + } catch (error) { + console.error(error) + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + } +} + +async function getTranslation({ + cid, + fromLang, + toLang = "en", +}: { + cid: string + fromLang?: Language + toLang?: Language +}): Promise { + if (fromLang === toLang) { + return undefined + } + + const translatedContent = (await cacheGet({ + key: ["translation", cid, toLang], + allowEmpty: true, + noUpdate: true, + noExpire: true, + getValueFun: async () => { + let result + + await lock.acquire(`translation_${cid}`, async () => { + const meta = await prisma.metadata.findFirst({ + where: { + uri: `ipfs://${cid}`, + }, + }) + + const key = "ai_translation" + const translations = meta?.[key as keyof Metadata] as Record< + string, + ContentTranslation + > + const translatedJson = translations?.[toLang] + + if (translatedJson) { + result = translatedJson + } else { + const newTranslation = await getOriginalTranslation(cid, toLang) + if (newTranslation) { + /** + * e.g. + * + * { + * "en": { + * "title": "title", + * "content": "content" + * }, + * "zh": { + * "title": "标题", + * "content": "内容" + * }, + * ... + * } + * + */ + const finalTranslation = { + ...translations, + [toLang]: newTranslation, + } + + if (meta) { + await prisma.metadata.update({ + where: { uri: `ipfs://${cid}` }, + data: { + [key as keyof Metadata]: finalTranslation, + }, + }) + } else { + await prisma.metadata.create({ + data: { + uri: `ipfs://${cid}`, + [key as keyof Metadata]: finalTranslation, + }, + }) + } + result = newTranslation + } + } + }) + + return result + }, + })) as ContentTranslation | undefined + + return translatedContent +} + +export async function GET(req: Request): Promise { + let { cid, fromLang, toLang } = getQuery(req) + + const res = new NextServerResponse() + + if (!cid) { + return res.status(400).send("Bad Request") + } + + return res.status(200).json({ + data: await getTranslation({ + cid, + fromLang, + toLang, + }), + }) +} diff --git a/src/components/home/HomeFeed.tsx b/src/components/home/HomeFeed.tsx index c72d26cc..0cff24d0 100644 --- a/src/components/home/HomeFeed.tsx +++ b/src/components/home/HomeFeed.tsx @@ -1,6 +1,6 @@ "use client" -import { useTranslations } from "next-intl" +import { useLocale, useTranslations } from "next-intl" import { useParams, useSearchParams } from "next/navigation" import { useEffect, useState } from "react" import { VirtuosoGrid } from "react-virtuoso" @@ -15,13 +15,14 @@ import { Tabs } from "~/components/ui/Tabs" import { Tooltip } from "~/components/ui/Tooltip" import { useIsMobileLayout } from "~/hooks/useMobileLayout" import { getStorage, setStorage } from "~/lib/storage" -import { ExpandedNote } from "~/lib/types" +import { ExpandedNote, Language } from "~/lib/types" import type { FeedType, SearchType } from "~/models/home.model" import { useGetFeed } from "~/queries/home" export const HomeFeed = ({ type }: { type?: FeedType }) => { const t = useTranslations() const searchParams = useSearchParams() + const locale = useLocale() as Language const currentCharacterId = useAccountState( (s) => s.computed.account?.characterId, @@ -43,6 +44,7 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => { let feedConfig: Parameters[0] = { type, } + switch (type) { case "following": feedConfig = { @@ -77,7 +79,10 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => { break } - const feed = useGetFeed(feedConfig) + const feed = useGetFeed({ + ...feedConfig, + translateTo: locale, + }) const hasFiltering = type === "latest" diff --git a/src/lib/expand-unit.ts b/src/lib/expand-unit.ts index ae0e3584..f11908de 100644 --- a/src/lib/expand-unit.ts +++ b/src/lib/expand-unit.ts @@ -5,7 +5,12 @@ import removeMarkdown from "remove-markdown" import { SCORE_API_DOMAIN, SITE_URL } from "~/lib/env" import { toCid, toGateway } from "~/lib/ipfs-parser" import readingTime from "~/lib/reading-time" -import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types" +import { + ExpandedCharacter, + ExpandedNote, + Language, + PortfolioStats, +} from "~/lib/types" import { detectLanguage } from "./detect-lang" import { getNoteSlug } from "./helpers" @@ -17,6 +22,7 @@ export const expandCrossbellNote = async ({ keyword, useHTML, disableAutofill, + translateTo, }: { note: NoteEntity & { metadata?: { @@ -30,6 +36,7 @@ export const expandCrossbellNote = async ({ keyword?: string useHTML?: boolean disableAutofill?: boolean + translateTo?: Language }) => { if (note.metadata?.uri && !note.metadata?.content) { note.metadata.content = await ( @@ -50,11 +57,41 @@ export const expandCrossbellNote = async ({ let rendered if (expandedNote.metadata?.content?.content) { const { renderPageContent } = await import("~/markdown") - const content = expandedNote.metadata.content.content - rendered = renderPageContent(content, true) + + const processedContent = removeMarkdown( + expandedNote.metadata.content.content.replace(/```[^]+?```/g, ""), + ) + const detectedLang = detectLanguage(processedContent) + expandedNote.metadata.content.originalLanguage = detectedLang + + if (translateTo) { + try { + const translation = await ( + await fetch( + `${SITE_URL}/api/translate-note?` + + new URLSearchParams({ + cid: toCid(expandedNote.metadata.uri || ""), + fromLang: expandedNote.metadata.content.originalLanguage, + toLang: translateTo, + } as any), + ) + ).json() + + if (translation.data) { + expandedNote.metadata.content.content = translation.data.content + expandedNote.metadata.content.title = translation.data.title + } + } catch (e) { + // do nothing + } + } + + rendered = renderPageContent(expandedNote.metadata.content.content!, true) if (keyword) { - const position = content.toLowerCase().indexOf(keyword.toLowerCase()) - expandedNote.metadata.content.summary = `...${content.slice( + const position = expandedNote.metadata.content + .content!.toLowerCase() + .indexOf(keyword.toLowerCase()) + expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content!.slice( position - 10, position + 100, )}` @@ -64,12 +101,6 @@ export const expandCrossbellNote = async ({ } } - const processedContent = removeMarkdown( - content.replace(/```[^]+?```/g, ""), - ) - const detectedLang = detectLanguage(processedContent) - expandedNote.metadata.content.originalLanguage = detectedLang - expandedNote.metadata.content.audio = rendered.audio expandedNote.metadata.content.frontMatter = rendered.frontMatter diff --git a/src/models/home.model.ts b/src/models/home.model.ts index fd53a78b..e4acfd54 100644 --- a/src/models/home.model.ts +++ b/src/models/home.model.ts @@ -6,7 +6,7 @@ import { gql } from "@urql/core" import countCharacters from "~/lib/character-count" import { expandCrossbellNote } from "~/lib/expand-unit" -import { ExpandedNote } from "~/lib/types" +import { ExpandedNote, Language } from "~/lib/types" import { client } from "~/queries/graphql" import filter from "../../data/filter.json" @@ -36,6 +36,7 @@ export async function getFeed({ tag, useHTML, topic, + translateTo, }: { type?: FeedType cursor?: string @@ -47,6 +48,7 @@ export async function getFeed({ tag?: string useHTML?: boolean topic?: string + translateTo?: Language }) { if (type === "search" && !searchKeyword) { type = "latest" @@ -141,6 +143,7 @@ export async function getFeed({ note: page, useScore: true, useHTML, + translateTo, }), ), ) @@ -235,6 +238,7 @@ export async function getFeed({ const expand = await expandCrossbellNote({ note: page, useHTML, + translateTo, }) if (expand.toNote) { if (expand.toNote.toNote) { @@ -251,6 +255,7 @@ export async function getFeed({ expand.toNote = await expandCrossbellNote({ note: expand.toNote, useHTML, + translateTo, }) delete expand.toNote.toNote } @@ -321,6 +326,7 @@ export async function getFeed({ useScore: true, useStat: true, useHTML, + translateTo, }), ), ) @@ -359,6 +365,7 @@ export async function getFeed({ expandCrossbellNote({ note: page, useHTML, + translateTo, }), ), ) @@ -465,6 +472,7 @@ export async function getFeed({ expandCrossbellNote({ note: page, useHTML, + translateTo, }), ), ) @@ -540,6 +548,7 @@ export async function getFeed({ expandCrossbellNote({ note: page, useHTML, + translateTo, }), ), ) @@ -644,6 +653,7 @@ export async function getFeed({ note: page, useHTML, useStat: true, + translateTo, }) return expand }, @@ -769,6 +779,7 @@ export async function getFeed({ note: page, useHTML, useStat: true, + translateTo, }) return expand }, @@ -813,6 +824,7 @@ export async function getFeed({ useScore: false, keyword: searchKeyword, useHTML, + translateTo, }), ), ) @@ -850,6 +862,7 @@ export async function getFeed({ useStat: false, useScore: true, useHTML, + translateTo, }), ), ) diff --git a/src/models/page.model.ts b/src/models/page.model.ts index b65ea0d0..22653aee 100644 --- a/src/models/page.model.ts +++ b/src/models/page.model.ts @@ -22,6 +22,7 @@ import { checkSlugReservedWords } from "~/lib/slug-reserved-words" import { getKeys, getStorage } from "~/lib/storage" import { ExpandedNote, + Language, NoteType, PagesSortTypes, PageVisibilityEnum, @@ -419,6 +420,7 @@ export async function getPage(input: { noteId?: number handle?: string // In order to be compatible with old drafts disableAutofill?: boolean + translateTo?: Language }) { const mustLocal = input.slug?.startsWith("!local-") && !input.noteId @@ -570,6 +572,7 @@ export async function getPage(input: { note: page, useStat: input.useStat, disableAutofill: input.disableAutofill, + translateTo: input.translateTo, }) } diff --git a/src/queries/page.server.ts b/src/queries/page.server.ts index c72a0b61..0211c3c4 100644 --- a/src/queries/page.server.ts +++ b/src/queries/page.server.ts @@ -1,20 +1,14 @@ import AsyncLock from "async-lock" -import { - AnalyzeDocumentChain, - LLMChain, - loadSummarizationChain, -} from "langchain/chains" +import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains" import { OpenAI } from "langchain/llms/openai" import { PromptTemplate } from "langchain/prompts" -import { getLocale } from "next-intl/server" import removeMarkdown from "remove-markdown" import { Metadata } from "@prisma/client" import { QueryClient } from "@tanstack/react-query" -import { defaultLocale, languageNames, locales } from "~/i18n" -import { toCid, toGateway } from "~/lib/ipfs-parser" -import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length" +import { defaultLocale, locales } from "~/i18n" +import { toGateway } from "~/lib/ipfs-parser" import prisma from "~/lib/prisma.server" import { cacheGet } from "~/lib/redis.server" import { Language } from "~/lib/types" @@ -38,6 +32,7 @@ export const fetchGetPage = async ( useStat: input.useStat, noteId: input.noteId, handle: input.handle, + translateTo: input.translateTo, }), }) as Promise> }) @@ -77,203 +72,6 @@ export const fetchGetPagesBySite = async ( }) } -// Content translation - -type ContentTranslation = { - title?: string - content?: string -} - -let translationModel4K: OpenAI | undefined -let translationModel16K: OpenAI | undefined -if (process.env.OPENAI_API_KEY) { - const options = { - openAIApiKey: process.env.OPENAI_API_KEY, - temperature: 0.2, - maxTokens: -1, - } - translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" }) - translationModel16K = new OpenAI({ - ...options, - modelName: "gpt-3.5-turbo-16k", - }) -} - -type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja" -const translationChains = new Map() - -const getOriginalTranslation = async ( - cid: string, - targetLang: Language, - originalLang?: Language, -): Promise => { - if (!translationModel4K || !translationModel16K) return - - try { - const { title, content } = await ( - await fetch(toGateway(`ipfs://${cid}`)) - ).json() - - // If the detected language is the same as the target language, return the original content - if (originalLang === targetLang) { - console.warn( - `|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`, - ) - return - } - - console.time(`fetching translation ${cid}, ${targetLang}`) - const { modelSize, tokens } = llmModelSwitcherByTextLength(content, { - includeResponse: { lang: targetLang }, - }) - - if (!modelSize) { - console.error( - `|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`, - ) - return - } - - let chain = translationChains.get(`${modelSize}_${targetLang}`) - - if (!chain) { - const prompt = new PromptTemplate({ - template: `Translate the following text into "${languageNames[targetLang]}" language: - {text} - Translation:`, - inputVariables: ["text"], - }) - - const translateModel = - modelSize === "4k" ? translationModel4K : translationModel16K - - chain = new LLMChain({ llm: translateModel, prompt }) - - translationChains.set(`${modelSize}_${targetLang}`, chain) - } - - const t = await chain.call({ text: title }) - const c = await chain.call({ text: content }) - - console.timeEnd(`fetching translation ${cid}, ${targetLang}`) - - return { - title: t.text, - content: c.text, - } - } catch (error) { - console.error(error) - console.timeEnd(`fetching translation ${cid}, ${targetLang}`) - } -} - -async function getTranslation({ - cid, - lang = "en", -}: { - cid: string - lang?: Language -}) { - const translatedContent = (await cacheGet({ - key: ["translation", cid, lang], - allowEmpty: true, - noUpdate: true, - noExpire: true, - getValueFun: async () => { - let result - - await lock.acquire(`translation_${cid}`, async () => { - const meta = await prisma.metadata.findFirst({ - where: { - uri: `ipfs://${cid}`, - }, - }) - - const key = "ai_translation" - const translations = meta?.[key as keyof Metadata] as Record< - string, - ContentTranslation - > - const translatedJson = translations?.[lang] - - if (translatedJson) { - result = translatedJson - } else { - const newTranslation = await getOriginalTranslation(cid, lang) - if (newTranslation) { - /** - * e.g. - * - * { - * "en": { - * "title": "title", - * "content": "content" - * }, - * "zh": { - * "title": "标题", - * "content": "内容" - * }, - * ... - * } - * - */ - const finalTranslation = { - ...translations, - [lang]: newTranslation, - } - - if (meta) { - await prisma.metadata.update({ - where: { uri: `ipfs://${cid}` }, - data: { - [key as keyof Metadata]: finalTranslation, - }, - }) - } else { - await prisma.metadata.create({ - data: { - uri: `ipfs://${cid}`, - [key as keyof Metadata]: finalTranslation, - }, - }) - } - result = newTranslation - } - } - }) - - return result - }, - })) as ContentTranslation | undefined - - return translatedContent -} - -// Modify page content with translation -export async function decoratePageWithTranslation( - page?: Awaited> | null, -) { - if (!page) return - const cid = toCid(page?.metadata?.uri || "") - - const targetLanguage = (await getLocale()) as Language - const originalLanguage = page?.metadata?.content?.originalLanguage - - if (originalLanguage === targetLanguage) { - return - } - - const translatedContent = await getTranslation({ - cid, - lang: targetLanguage, - }) - - if (translatedContent && page?.metadata?.content) { - page.metadata.content["content"] = translatedContent.content - page.metadata.content["title"] = translatedContent.title - } -} - // Post summary let model: OpenAI | undefined