feat: Invoke translateNote via API.
This commit is contained in:
@@ -16,6 +16,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser"
|
||||
import { isInRN } from "~/lib/is-in-rn"
|
||||
import { isOnlyContent } from "~/lib/is-only-content"
|
||||
import getQueryClient from "~/lib/query-client"
|
||||
import { Language } from "~/lib/types"
|
||||
import { fetchGetPage, getSummary } from "~/queries/page.server"
|
||||
import { fetchGetSite } from "~/queries/site.server"
|
||||
|
||||
@@ -28,7 +29,7 @@ export async function generateMetadata({
|
||||
}
|
||||
}): Promise<Metadata> {
|
||||
const queryClient = getQueryClient()
|
||||
|
||||
const locale = (await getLocale()) as Language
|
||||
const site = await fetchGetSite(params.site, queryClient)
|
||||
|
||||
const page = await fetchGetPage(
|
||||
@@ -36,12 +37,11 @@ export async function generateMetadata({
|
||||
characterId: site?.characterId,
|
||||
slug: params.slug,
|
||||
useStat: true,
|
||||
translateTo: locale,
|
||||
},
|
||||
queryClient,
|
||||
)
|
||||
|
||||
// await decoratePageWithTranslation(page)
|
||||
|
||||
const title = `${
|
||||
page?.metadata?.content?.title || page?.metadata?.content?.content
|
||||
} - ${site?.metadata?.content?.name || site?.handle}`
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { getQuery, NextServerResponse } from "~/lib/server-helper"
|
||||
import { getFeed } from "~/models/home.model"
|
||||
import { decoratePageWithTranslation } from "~/queries/page.server"
|
||||
|
||||
export async function GET(req: Request) {
|
||||
const query = getQuery(req)
|
||||
@@ -16,12 +15,9 @@ export async function GET(req: Request) {
|
||||
tag: query.tag,
|
||||
useHTML: false,
|
||||
topic: query.topic,
|
||||
translateTo: query.translateTo,
|
||||
})
|
||||
|
||||
for (const item of result.list) {
|
||||
await decoratePageWithTranslation(item)
|
||||
}
|
||||
|
||||
const res = new NextServerResponse()
|
||||
return res.status(200).json(result)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
import AsyncLock from "async-lock"
|
||||
import { LLMChain } from "langchain/chains"
|
||||
import { OpenAI } from "langchain/llms/openai"
|
||||
import { PromptTemplate } from "langchain/prompts"
|
||||
|
||||
import { Metadata } from "@prisma/client"
|
||||
|
||||
import { languageNames } from "~/i18n"
|
||||
import { toGateway } from "~/lib/ipfs-parser"
|
||||
import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
|
||||
import prisma from "~/lib/prisma.server"
|
||||
import { cacheGet } from "~/lib/redis.server"
|
||||
import { getQuery, NextServerResponse } from "~/lib/server-helper"
|
||||
import { Language } from "~/lib/types"
|
||||
|
||||
const lock = new AsyncLock()
|
||||
|
||||
type ContentTranslation = {
|
||||
title?: string
|
||||
content?: string
|
||||
}
|
||||
|
||||
let translationModel4K: OpenAI | undefined
|
||||
let translationModel16K: OpenAI | undefined
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
const options = {
|
||||
openAIApiKey: process.env.OPENAI_API_KEY,
|
||||
temperature: 0.2,
|
||||
maxTokens: -1,
|
||||
}
|
||||
translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
|
||||
translationModel16K = new OpenAI({
|
||||
...options,
|
||||
modelName: "gpt-3.5-turbo-16k",
|
||||
})
|
||||
}
|
||||
|
||||
type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
|
||||
const translationChains = new Map<ChainKeyType, LLMChain>()
|
||||
|
||||
const getOriginalTranslation = async (
|
||||
cid: string,
|
||||
targetLang: Language,
|
||||
originalLang?: Language,
|
||||
): Promise<ContentTranslation | undefined> => {
|
||||
if (!translationModel4K || !translationModel16K) return
|
||||
|
||||
try {
|
||||
const { title, content } = await (
|
||||
await fetch(toGateway(`ipfs://${cid}`))
|
||||
).json()
|
||||
|
||||
// If the detected language is the same as the target language, return the original content
|
||||
if (originalLang === targetLang) {
|
||||
console.warn(
|
||||
`|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
console.time(`fetching translation ${cid}, ${targetLang}`)
|
||||
const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
|
||||
includeResponse: { lang: targetLang },
|
||||
})
|
||||
|
||||
if (!modelSize) {
|
||||
console.error(
|
||||
`|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
let chain = translationChains.get(`${modelSize}_${targetLang}`)
|
||||
|
||||
if (!chain) {
|
||||
const prompt = new PromptTemplate({
|
||||
template: `Translate the following text into "${languageNames[targetLang]}" language:
|
||||
{text}
|
||||
Translation:`,
|
||||
inputVariables: ["text"],
|
||||
})
|
||||
|
||||
const translateModel =
|
||||
modelSize === "4k" ? translationModel4K : translationModel16K
|
||||
|
||||
chain = new LLMChain({ llm: translateModel, prompt })
|
||||
|
||||
translationChains.set(`${modelSize}_${targetLang}`, chain)
|
||||
}
|
||||
|
||||
const t = await chain.call({ text: title })
|
||||
const c = await chain.call({ text: content })
|
||||
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
|
||||
return {
|
||||
title: t.text,
|
||||
content: c.text,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function getTranslation({
|
||||
cid,
|
||||
fromLang,
|
||||
toLang = "en",
|
||||
}: {
|
||||
cid: string
|
||||
fromLang?: Language
|
||||
toLang?: Language
|
||||
}): Promise<ContentTranslation | undefined> {
|
||||
if (fromLang === toLang) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const translatedContent = (await cacheGet({
|
||||
key: ["translation", cid, toLang],
|
||||
allowEmpty: true,
|
||||
noUpdate: true,
|
||||
noExpire: true,
|
||||
getValueFun: async () => {
|
||||
let result
|
||||
|
||||
await lock.acquire(`translation_${cid}`, async () => {
|
||||
const meta = await prisma.metadata.findFirst({
|
||||
where: {
|
||||
uri: `ipfs://${cid}`,
|
||||
},
|
||||
})
|
||||
|
||||
const key = "ai_translation"
|
||||
const translations = meta?.[key as keyof Metadata] as Record<
|
||||
string,
|
||||
ContentTranslation
|
||||
>
|
||||
const translatedJson = translations?.[toLang]
|
||||
|
||||
if (translatedJson) {
|
||||
result = translatedJson
|
||||
} else {
|
||||
const newTranslation = await getOriginalTranslation(cid, toLang)
|
||||
if (newTranslation) {
|
||||
/**
|
||||
* e.g.
|
||||
*
|
||||
* {
|
||||
* "en": {
|
||||
* "title": "title",
|
||||
* "content": "content"
|
||||
* },
|
||||
* "zh": {
|
||||
* "title": "标题",
|
||||
* "content": "内容"
|
||||
* },
|
||||
* ...
|
||||
* }
|
||||
*
|
||||
*/
|
||||
const finalTranslation = {
|
||||
...translations,
|
||||
[toLang]: newTranslation,
|
||||
}
|
||||
|
||||
if (meta) {
|
||||
await prisma.metadata.update({
|
||||
where: { uri: `ipfs://${cid}` },
|
||||
data: {
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await prisma.metadata.create({
|
||||
data: {
|
||||
uri: `ipfs://${cid}`,
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
}
|
||||
result = newTranslation
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
},
|
||||
})) as ContentTranslation | undefined
|
||||
|
||||
return translatedContent
|
||||
}
|
||||
|
||||
export async function GET(req: Request): Promise<Response> {
|
||||
let { cid, fromLang, toLang } = getQuery(req)
|
||||
|
||||
const res = new NextServerResponse()
|
||||
|
||||
if (!cid) {
|
||||
return res.status(400).send("Bad Request")
|
||||
}
|
||||
|
||||
return res.status(200).json({
|
||||
data: await getTranslation({
|
||||
cid,
|
||||
fromLang,
|
||||
toLang,
|
||||
}),
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
"use client"
|
||||
|
||||
import { useTranslations } from "next-intl"
|
||||
import { useLocale, useTranslations } from "next-intl"
|
||||
import { useParams, useSearchParams } from "next/navigation"
|
||||
import { useEffect, useState } from "react"
|
||||
import { VirtuosoGrid } from "react-virtuoso"
|
||||
@@ -15,13 +15,14 @@ import { Tabs } from "~/components/ui/Tabs"
|
||||
import { Tooltip } from "~/components/ui/Tooltip"
|
||||
import { useIsMobileLayout } from "~/hooks/useMobileLayout"
|
||||
import { getStorage, setStorage } from "~/lib/storage"
|
||||
import { ExpandedNote } from "~/lib/types"
|
||||
import { ExpandedNote, Language } from "~/lib/types"
|
||||
import type { FeedType, SearchType } from "~/models/home.model"
|
||||
import { useGetFeed } from "~/queries/home"
|
||||
|
||||
export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
||||
const t = useTranslations()
|
||||
const searchParams = useSearchParams()
|
||||
const locale = useLocale() as Language
|
||||
|
||||
const currentCharacterId = useAccountState(
|
||||
(s) => s.computed.account?.characterId,
|
||||
@@ -43,6 +44,7 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
||||
let feedConfig: Parameters<typeof useGetFeed>[0] = {
|
||||
type,
|
||||
}
|
||||
|
||||
switch (type) {
|
||||
case "following":
|
||||
feedConfig = {
|
||||
@@ -77,7 +79,10 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
|
||||
break
|
||||
}
|
||||
|
||||
const feed = useGetFeed(feedConfig)
|
||||
const feed = useGetFeed({
|
||||
...feedConfig,
|
||||
translateTo: locale,
|
||||
})
|
||||
|
||||
const hasFiltering = type === "latest"
|
||||
|
||||
|
||||
+42
-11
@@ -5,7 +5,12 @@ import removeMarkdown from "remove-markdown"
|
||||
import { SCORE_API_DOMAIN, SITE_URL } from "~/lib/env"
|
||||
import { toCid, toGateway } from "~/lib/ipfs-parser"
|
||||
import readingTime from "~/lib/reading-time"
|
||||
import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types"
|
||||
import {
|
||||
ExpandedCharacter,
|
||||
ExpandedNote,
|
||||
Language,
|
||||
PortfolioStats,
|
||||
} from "~/lib/types"
|
||||
|
||||
import { detectLanguage } from "./detect-lang"
|
||||
import { getNoteSlug } from "./helpers"
|
||||
@@ -17,6 +22,7 @@ export const expandCrossbellNote = async ({
|
||||
keyword,
|
||||
useHTML,
|
||||
disableAutofill,
|
||||
translateTo,
|
||||
}: {
|
||||
note: NoteEntity & {
|
||||
metadata?: {
|
||||
@@ -30,6 +36,7 @@ export const expandCrossbellNote = async ({
|
||||
keyword?: string
|
||||
useHTML?: boolean
|
||||
disableAutofill?: boolean
|
||||
translateTo?: Language
|
||||
}) => {
|
||||
if (note.metadata?.uri && !note.metadata?.content) {
|
||||
note.metadata.content = await (
|
||||
@@ -50,11 +57,41 @@ export const expandCrossbellNote = async ({
|
||||
let rendered
|
||||
if (expandedNote.metadata?.content?.content) {
|
||||
const { renderPageContent } = await import("~/markdown")
|
||||
const content = expandedNote.metadata.content.content
|
||||
rendered = renderPageContent(content, true)
|
||||
|
||||
const processedContent = removeMarkdown(
|
||||
expandedNote.metadata.content.content.replace(/```[^]+?```/g, ""),
|
||||
)
|
||||
const detectedLang = detectLanguage(processedContent)
|
||||
expandedNote.metadata.content.originalLanguage = detectedLang
|
||||
|
||||
if (translateTo) {
|
||||
try {
|
||||
const translation = await (
|
||||
await fetch(
|
||||
`${SITE_URL}/api/translate-note?` +
|
||||
new URLSearchParams({
|
||||
cid: toCid(expandedNote.metadata.uri || ""),
|
||||
fromLang: expandedNote.metadata.content.originalLanguage,
|
||||
toLang: translateTo,
|
||||
} as any),
|
||||
)
|
||||
).json()
|
||||
|
||||
if (translation.data) {
|
||||
expandedNote.metadata.content.content = translation.data.content
|
||||
expandedNote.metadata.content.title = translation.data.title
|
||||
}
|
||||
} catch (e) {
|
||||
// do nothing
|
||||
}
|
||||
}
|
||||
|
||||
rendered = renderPageContent(expandedNote.metadata.content.content!, true)
|
||||
if (keyword) {
|
||||
const position = content.toLowerCase().indexOf(keyword.toLowerCase())
|
||||
expandedNote.metadata.content.summary = `...${content.slice(
|
||||
const position = expandedNote.metadata.content
|
||||
.content!.toLowerCase()
|
||||
.indexOf(keyword.toLowerCase())
|
||||
expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content!.slice(
|
||||
position - 10,
|
||||
position + 100,
|
||||
)}`
|
||||
@@ -64,12 +101,6 @@ export const expandCrossbellNote = async ({
|
||||
}
|
||||
}
|
||||
|
||||
const processedContent = removeMarkdown(
|
||||
content.replace(/```[^]+?```/g, ""),
|
||||
)
|
||||
const detectedLang = detectLanguage(processedContent)
|
||||
expandedNote.metadata.content.originalLanguage = detectedLang
|
||||
|
||||
expandedNote.metadata.content.audio = rendered.audio
|
||||
expandedNote.metadata.content.frontMatter = rendered.frontMatter
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import { gql } from "@urql/core"
|
||||
|
||||
import countCharacters from "~/lib/character-count"
|
||||
import { expandCrossbellNote } from "~/lib/expand-unit"
|
||||
import { ExpandedNote } from "~/lib/types"
|
||||
import { ExpandedNote, Language } from "~/lib/types"
|
||||
import { client } from "~/queries/graphql"
|
||||
|
||||
import filter from "../../data/filter.json"
|
||||
@@ -36,6 +36,7 @@ export async function getFeed({
|
||||
tag,
|
||||
useHTML,
|
||||
topic,
|
||||
translateTo,
|
||||
}: {
|
||||
type?: FeedType
|
||||
cursor?: string
|
||||
@@ -47,6 +48,7 @@ export async function getFeed({
|
||||
tag?: string
|
||||
useHTML?: boolean
|
||||
topic?: string
|
||||
translateTo?: Language
|
||||
}) {
|
||||
if (type === "search" && !searchKeyword) {
|
||||
type = "latest"
|
||||
@@ -141,6 +143,7 @@ export async function getFeed({
|
||||
note: page,
|
||||
useScore: true,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -235,6 +238,7 @@ export async function getFeed({
|
||||
const expand = await expandCrossbellNote({
|
||||
note: page,
|
||||
useHTML,
|
||||
translateTo,
|
||||
})
|
||||
if (expand.toNote) {
|
||||
if (expand.toNote.toNote) {
|
||||
@@ -251,6 +255,7 @@ export async function getFeed({
|
||||
expand.toNote = await expandCrossbellNote({
|
||||
note: expand.toNote,
|
||||
useHTML,
|
||||
translateTo,
|
||||
})
|
||||
delete expand.toNote.toNote
|
||||
}
|
||||
@@ -321,6 +326,7 @@ export async function getFeed({
|
||||
useScore: true,
|
||||
useStat: true,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -359,6 +365,7 @@ export async function getFeed({
|
||||
expandCrossbellNote({
|
||||
note: page,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -465,6 +472,7 @@ export async function getFeed({
|
||||
expandCrossbellNote({
|
||||
note: page,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -540,6 +548,7 @@ export async function getFeed({
|
||||
expandCrossbellNote({
|
||||
note: page,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -644,6 +653,7 @@ export async function getFeed({
|
||||
note: page,
|
||||
useHTML,
|
||||
useStat: true,
|
||||
translateTo,
|
||||
})
|
||||
return expand
|
||||
},
|
||||
@@ -769,6 +779,7 @@ export async function getFeed({
|
||||
note: page,
|
||||
useHTML,
|
||||
useStat: true,
|
||||
translateTo,
|
||||
})
|
||||
return expand
|
||||
},
|
||||
@@ -813,6 +824,7 @@ export async function getFeed({
|
||||
useScore: false,
|
||||
keyword: searchKeyword,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
@@ -850,6 +862,7 @@ export async function getFeed({
|
||||
useStat: false,
|
||||
useScore: true,
|
||||
useHTML,
|
||||
translateTo,
|
||||
}),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ import { checkSlugReservedWords } from "~/lib/slug-reserved-words"
|
||||
import { getKeys, getStorage } from "~/lib/storage"
|
||||
import {
|
||||
ExpandedNote,
|
||||
Language,
|
||||
NoteType,
|
||||
PagesSortTypes,
|
||||
PageVisibilityEnum,
|
||||
@@ -419,6 +420,7 @@ export async function getPage<TRender extends boolean = false>(input: {
|
||||
noteId?: number
|
||||
handle?: string // In order to be compatible with old drafts
|
||||
disableAutofill?: boolean
|
||||
translateTo?: Language
|
||||
}) {
|
||||
const mustLocal = input.slug?.startsWith("!local-") && !input.noteId
|
||||
|
||||
@@ -570,6 +572,7 @@ export async function getPage<TRender extends boolean = false>(input: {
|
||||
note: page,
|
||||
useStat: input.useStat,
|
||||
disableAutofill: input.disableAutofill,
|
||||
translateTo: input.translateTo,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+4
-206
@@ -1,20 +1,14 @@
|
||||
import AsyncLock from "async-lock"
|
||||
import {
|
||||
AnalyzeDocumentChain,
|
||||
LLMChain,
|
||||
loadSummarizationChain,
|
||||
} from "langchain/chains"
|
||||
import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains"
|
||||
import { OpenAI } from "langchain/llms/openai"
|
||||
import { PromptTemplate } from "langchain/prompts"
|
||||
import { getLocale } from "next-intl/server"
|
||||
import removeMarkdown from "remove-markdown"
|
||||
|
||||
import { Metadata } from "@prisma/client"
|
||||
import { QueryClient } from "@tanstack/react-query"
|
||||
|
||||
import { defaultLocale, languageNames, locales } from "~/i18n"
|
||||
import { toCid, toGateway } from "~/lib/ipfs-parser"
|
||||
import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
|
||||
import { defaultLocale, locales } from "~/i18n"
|
||||
import { toGateway } from "~/lib/ipfs-parser"
|
||||
import prisma from "~/lib/prisma.server"
|
||||
import { cacheGet } from "~/lib/redis.server"
|
||||
import { Language } from "~/lib/types"
|
||||
@@ -38,6 +32,7 @@ export const fetchGetPage = async (
|
||||
useStat: input.useStat,
|
||||
noteId: input.noteId,
|
||||
handle: input.handle,
|
||||
translateTo: input.translateTo,
|
||||
}),
|
||||
}) as Promise<ReturnType<typeof pageModel.getPage>>
|
||||
})
|
||||
@@ -77,203 +72,6 @@ export const fetchGetPagesBySite = async (
|
||||
})
|
||||
}
|
||||
|
||||
// Content translation
|
||||
|
||||
type ContentTranslation = {
|
||||
title?: string
|
||||
content?: string
|
||||
}
|
||||
|
||||
let translationModel4K: OpenAI | undefined
|
||||
let translationModel16K: OpenAI | undefined
|
||||
if (process.env.OPENAI_API_KEY) {
|
||||
const options = {
|
||||
openAIApiKey: process.env.OPENAI_API_KEY,
|
||||
temperature: 0.2,
|
||||
maxTokens: -1,
|
||||
}
|
||||
translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
|
||||
translationModel16K = new OpenAI({
|
||||
...options,
|
||||
modelName: "gpt-3.5-turbo-16k",
|
||||
})
|
||||
}
|
||||
|
||||
type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
|
||||
const translationChains = new Map<ChainKeyType, LLMChain>()
|
||||
|
||||
const getOriginalTranslation = async (
|
||||
cid: string,
|
||||
targetLang: Language,
|
||||
originalLang?: Language,
|
||||
): Promise<ContentTranslation | undefined> => {
|
||||
if (!translationModel4K || !translationModel16K) return
|
||||
|
||||
try {
|
||||
const { title, content } = await (
|
||||
await fetch(toGateway(`ipfs://${cid}`))
|
||||
).json()
|
||||
|
||||
// If the detected language is the same as the target language, return the original content
|
||||
if (originalLang === targetLang) {
|
||||
console.warn(
|
||||
`|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
console.time(`fetching translation ${cid}, ${targetLang}`)
|
||||
const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
|
||||
includeResponse: { lang: targetLang },
|
||||
})
|
||||
|
||||
if (!modelSize) {
|
||||
console.error(
|
||||
`|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
let chain = translationChains.get(`${modelSize}_${targetLang}`)
|
||||
|
||||
if (!chain) {
|
||||
const prompt = new PromptTemplate({
|
||||
template: `Translate the following text into "${languageNames[targetLang]}" language:
|
||||
{text}
|
||||
Translation:`,
|
||||
inputVariables: ["text"],
|
||||
})
|
||||
|
||||
const translateModel =
|
||||
modelSize === "4k" ? translationModel4K : translationModel16K
|
||||
|
||||
chain = new LLMChain({ llm: translateModel, prompt })
|
||||
|
||||
translationChains.set(`${modelSize}_${targetLang}`, chain)
|
||||
}
|
||||
|
||||
const t = await chain.call({ text: title })
|
||||
const c = await chain.call({ text: content })
|
||||
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
|
||||
return {
|
||||
title: t.text,
|
||||
content: c.text,
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
|
||||
}
|
||||
}
|
||||
|
||||
async function getTranslation({
|
||||
cid,
|
||||
lang = "en",
|
||||
}: {
|
||||
cid: string
|
||||
lang?: Language
|
||||
}) {
|
||||
const translatedContent = (await cacheGet({
|
||||
key: ["translation", cid, lang],
|
||||
allowEmpty: true,
|
||||
noUpdate: true,
|
||||
noExpire: true,
|
||||
getValueFun: async () => {
|
||||
let result
|
||||
|
||||
await lock.acquire(`translation_${cid}`, async () => {
|
||||
const meta = await prisma.metadata.findFirst({
|
||||
where: {
|
||||
uri: `ipfs://${cid}`,
|
||||
},
|
||||
})
|
||||
|
||||
const key = "ai_translation"
|
||||
const translations = meta?.[key as keyof Metadata] as Record<
|
||||
string,
|
||||
ContentTranslation
|
||||
>
|
||||
const translatedJson = translations?.[lang]
|
||||
|
||||
if (translatedJson) {
|
||||
result = translatedJson
|
||||
} else {
|
||||
const newTranslation = await getOriginalTranslation(cid, lang)
|
||||
if (newTranslation) {
|
||||
/**
|
||||
* e.g.
|
||||
*
|
||||
* {
|
||||
* "en": {
|
||||
* "title": "title",
|
||||
* "content": "content"
|
||||
* },
|
||||
* "zh": {
|
||||
* "title": "标题",
|
||||
* "content": "内容"
|
||||
* },
|
||||
* ...
|
||||
* }
|
||||
*
|
||||
*/
|
||||
const finalTranslation = {
|
||||
...translations,
|
||||
[lang]: newTranslation,
|
||||
}
|
||||
|
||||
if (meta) {
|
||||
await prisma.metadata.update({
|
||||
where: { uri: `ipfs://${cid}` },
|
||||
data: {
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
await prisma.metadata.create({
|
||||
data: {
|
||||
uri: `ipfs://${cid}`,
|
||||
[key as keyof Metadata]: finalTranslation,
|
||||
},
|
||||
})
|
||||
}
|
||||
result = newTranslation
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return result
|
||||
},
|
||||
})) as ContentTranslation | undefined
|
||||
|
||||
return translatedContent
|
||||
}
|
||||
|
||||
// Modify page content with translation
|
||||
export async function decoratePageWithTranslation(
|
||||
page?: Awaited<ReturnType<typeof pageModel.getPage>> | null,
|
||||
) {
|
||||
if (!page) return
|
||||
const cid = toCid(page?.metadata?.uri || "")
|
||||
|
||||
const targetLanguage = (await getLocale()) as Language
|
||||
const originalLanguage = page?.metadata?.content?.originalLanguage
|
||||
|
||||
if (originalLanguage === targetLanguage) {
|
||||
return
|
||||
}
|
||||
|
||||
const translatedContent = await getTranslation({
|
||||
cid,
|
||||
lang: targetLanguage,
|
||||
})
|
||||
|
||||
if (translatedContent && page?.metadata?.content) {
|
||||
page.metadata.content["content"] = translatedContent.content
|
||||
page.metadata.content["title"] = translatedContent.title
|
||||
}
|
||||
}
|
||||
|
||||
// Post summary
|
||||
|
||||
let model: OpenAI | undefined
|
||||
|
||||
Reference in New Issue
Block a user