feat: Invoke translateNote via API.

This commit is contained in:
Caspian
2023-12-01 17:53:49 +00:00
parent f520e63b19
commit e7409a4e7a
8 changed files with 285 additions and 229 deletions
+3 -3
View File
@@ -16,6 +16,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser"
import { isInRN } from "~/lib/is-in-rn"
import { isOnlyContent } from "~/lib/is-only-content"
import getQueryClient from "~/lib/query-client"
import { Language } from "~/lib/types"
import { fetchGetPage, getSummary } from "~/queries/page.server"
import { fetchGetSite } from "~/queries/site.server"
@@ -28,7 +29,7 @@ export async function generateMetadata({
}
}): Promise<Metadata> {
const queryClient = getQueryClient()
const locale = (await getLocale()) as Language
const site = await fetchGetSite(params.site, queryClient)
const page = await fetchGetPage(
@@ -36,12 +37,11 @@ export async function generateMetadata({
characterId: site?.characterId,
slug: params.slug,
useStat: true,
translateTo: locale,
},
queryClient,
)
// await decoratePageWithTranslation(page)
const title = `${
page?.metadata?.content?.title || page?.metadata?.content?.content
} - ${site?.metadata?.content?.name || site?.handle}`
+1 -5
View File
@@ -1,6 +1,5 @@
import { getQuery, NextServerResponse } from "~/lib/server-helper"
import { getFeed } from "~/models/home.model"
import { decoratePageWithTranslation } from "~/queries/page.server"
export async function GET(req: Request) {
const query = getQuery(req)
@@ -16,12 +15,9 @@ export async function GET(req: Request) {
tag: query.tag,
useHTML: false,
topic: query.topic,
translateTo: query.translateTo,
})
for (const item of result.list) {
await decoratePageWithTranslation(item)
}
const res = new NextServerResponse()
return res.status(200).json(result)
}
+210
View File
@@ -0,0 +1,210 @@
import AsyncLock from "async-lock"
import { LLMChain } from "langchain/chains"
import { OpenAI } from "langchain/llms/openai"
import { PromptTemplate } from "langchain/prompts"
import { Metadata } from "@prisma/client"
import { languageNames } from "~/i18n"
import { toGateway } from "~/lib/ipfs-parser"
import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
import prisma from "~/lib/prisma.server"
import { cacheGet } from "~/lib/redis.server"
import { getQuery, NextServerResponse } from "~/lib/server-helper"
import { Language } from "~/lib/types"
const lock = new AsyncLock()
type ContentTranslation = {
title?: string
content?: string
}
let translationModel4K: OpenAI | undefined
let translationModel16K: OpenAI | undefined
if (process.env.OPENAI_API_KEY) {
const options = {
openAIApiKey: process.env.OPENAI_API_KEY,
temperature: 0.2,
maxTokens: -1,
}
translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
translationModel16K = new OpenAI({
...options,
modelName: "gpt-3.5-turbo-16k",
})
}
type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
const translationChains = new Map<ChainKeyType, LLMChain>()
const getOriginalTranslation = async (
cid: string,
targetLang: Language,
originalLang?: Language,
): Promise<ContentTranslation | undefined> => {
if (!translationModel4K || !translationModel16K) return
try {
const { title, content } = await (
await fetch(toGateway(`ipfs://${cid}`))
).json()
// If the detected language is the same as the target language, return the original content
if (originalLang === targetLang) {
console.warn(
`|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
)
return
}
console.time(`fetching translation ${cid}, ${targetLang}`)
const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
includeResponse: { lang: targetLang },
})
if (!modelSize) {
console.error(
`|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
)
return
}
let chain = translationChains.get(`${modelSize}_${targetLang}`)
if (!chain) {
const prompt = new PromptTemplate({
template: `Translate the following text into "${languageNames[targetLang]}" language:
{text}
Translation:`,
inputVariables: ["text"],
})
const translateModel =
modelSize === "4k" ? translationModel4K : translationModel16K
chain = new LLMChain({ llm: translateModel, prompt })
translationChains.set(`${modelSize}_${targetLang}`, chain)
}
const t = await chain.call({ text: title })
const c = await chain.call({ text: content })
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
return {
title: t.text,
content: c.text,
}
} catch (error) {
console.error(error)
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
}
}
async function getTranslation({
cid,
fromLang,
toLang = "en",
}: {
cid: string
fromLang?: Language
toLang?: Language
}): Promise<ContentTranslation | undefined> {
if (fromLang === toLang) {
return undefined
}
const translatedContent = (await cacheGet({
key: ["translation", cid, toLang],
allowEmpty: true,
noUpdate: true,
noExpire: true,
getValueFun: async () => {
let result
await lock.acquire(`translation_${cid}`, async () => {
const meta = await prisma.metadata.findFirst({
where: {
uri: `ipfs://${cid}`,
},
})
const key = "ai_translation"
const translations = meta?.[key as keyof Metadata] as Record<
string,
ContentTranslation
>
const translatedJson = translations?.[toLang]
if (translatedJson) {
result = translatedJson
} else {
const newTranslation = await getOriginalTranslation(cid, toLang)
if (newTranslation) {
/**
* e.g.
*
* {
* "en": {
* "title": "title",
* "content": "content"
* },
* "zh": {
* "title": "标题",
* "content": "内容"
* },
* ...
* }
*
*/
const finalTranslation = {
...translations,
[toLang]: newTranslation,
}
if (meta) {
await prisma.metadata.update({
where: { uri: `ipfs://${cid}` },
data: {
[key as keyof Metadata]: finalTranslation,
},
})
} else {
await prisma.metadata.create({
data: {
uri: `ipfs://${cid}`,
[key as keyof Metadata]: finalTranslation,
},
})
}
result = newTranslation
}
}
})
return result
},
})) as ContentTranslation | undefined
return translatedContent
}
export async function GET(req: Request): Promise<Response> {
let { cid, fromLang, toLang } = getQuery(req)
const res = new NextServerResponse()
if (!cid) {
return res.status(400).send("Bad Request")
}
return res.status(200).json({
data: await getTranslation({
cid,
fromLang,
toLang,
}),
})
}
+8 -3
View File
@@ -1,6 +1,6 @@
"use client"
import { useTranslations } from "next-intl"
import { useLocale, useTranslations } from "next-intl"
import { useParams, useSearchParams } from "next/navigation"
import { useEffect, useState } from "react"
import { VirtuosoGrid } from "react-virtuoso"
@@ -15,13 +15,14 @@ import { Tabs } from "~/components/ui/Tabs"
import { Tooltip } from "~/components/ui/Tooltip"
import { useIsMobileLayout } from "~/hooks/useMobileLayout"
import { getStorage, setStorage } from "~/lib/storage"
import { ExpandedNote } from "~/lib/types"
import { ExpandedNote, Language } from "~/lib/types"
import type { FeedType, SearchType } from "~/models/home.model"
import { useGetFeed } from "~/queries/home"
export const HomeFeed = ({ type }: { type?: FeedType }) => {
const t = useTranslations()
const searchParams = useSearchParams()
const locale = useLocale() as Language
const currentCharacterId = useAccountState(
(s) => s.computed.account?.characterId,
@@ -43,6 +44,7 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
let feedConfig: Parameters<typeof useGetFeed>[0] = {
type,
}
switch (type) {
case "following":
feedConfig = {
@@ -77,7 +79,10 @@ export const HomeFeed = ({ type }: { type?: FeedType }) => {
break
}
const feed = useGetFeed(feedConfig)
const feed = useGetFeed({
...feedConfig,
translateTo: locale,
})
const hasFiltering = type === "latest"
+42 -11
View File
@@ -5,7 +5,12 @@ import removeMarkdown from "remove-markdown"
import { SCORE_API_DOMAIN, SITE_URL } from "~/lib/env"
import { toCid, toGateway } from "~/lib/ipfs-parser"
import readingTime from "~/lib/reading-time"
import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types"
import {
ExpandedCharacter,
ExpandedNote,
Language,
PortfolioStats,
} from "~/lib/types"
import { detectLanguage } from "./detect-lang"
import { getNoteSlug } from "./helpers"
@@ -17,6 +22,7 @@ export const expandCrossbellNote = async ({
keyword,
useHTML,
disableAutofill,
translateTo,
}: {
note: NoteEntity & {
metadata?: {
@@ -30,6 +36,7 @@ export const expandCrossbellNote = async ({
keyword?: string
useHTML?: boolean
disableAutofill?: boolean
translateTo?: Language
}) => {
if (note.metadata?.uri && !note.metadata?.content) {
note.metadata.content = await (
@@ -50,11 +57,41 @@ export const expandCrossbellNote = async ({
let rendered
if (expandedNote.metadata?.content?.content) {
const { renderPageContent } = await import("~/markdown")
const content = expandedNote.metadata.content.content
rendered = renderPageContent(content, true)
const processedContent = removeMarkdown(
expandedNote.metadata.content.content.replace(/```[^]+?```/g, ""),
)
const detectedLang = detectLanguage(processedContent)
expandedNote.metadata.content.originalLanguage = detectedLang
if (translateTo) {
try {
const translation = await (
await fetch(
`${SITE_URL}/api/translate-note?` +
new URLSearchParams({
cid: toCid(expandedNote.metadata.uri || ""),
fromLang: expandedNote.metadata.content.originalLanguage,
toLang: translateTo,
} as any),
)
).json()
if (translation.data) {
expandedNote.metadata.content.content = translation.data.content
expandedNote.metadata.content.title = translation.data.title
}
} catch (e) {
// do nothing
}
}
rendered = renderPageContent(expandedNote.metadata.content.content!, true)
if (keyword) {
const position = content.toLowerCase().indexOf(keyword.toLowerCase())
expandedNote.metadata.content.summary = `...${content.slice(
const position = expandedNote.metadata.content
.content!.toLowerCase()
.indexOf(keyword.toLowerCase())
expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content!.slice(
position - 10,
position + 100,
)}`
@@ -64,12 +101,6 @@ export const expandCrossbellNote = async ({
}
}
const processedContent = removeMarkdown(
content.replace(/```[^]+?```/g, ""),
)
const detectedLang = detectLanguage(processedContent)
expandedNote.metadata.content.originalLanguage = detectedLang
expandedNote.metadata.content.audio = rendered.audio
expandedNote.metadata.content.frontMatter = rendered.frontMatter
+14 -1
View File
@@ -6,7 +6,7 @@ import { gql } from "@urql/core"
import countCharacters from "~/lib/character-count"
import { expandCrossbellNote } from "~/lib/expand-unit"
import { ExpandedNote } from "~/lib/types"
import { ExpandedNote, Language } from "~/lib/types"
import { client } from "~/queries/graphql"
import filter from "../../data/filter.json"
@@ -36,6 +36,7 @@ export async function getFeed({
tag,
useHTML,
topic,
translateTo,
}: {
type?: FeedType
cursor?: string
@@ -47,6 +48,7 @@ export async function getFeed({
tag?: string
useHTML?: boolean
topic?: string
translateTo?: Language
}) {
if (type === "search" && !searchKeyword) {
type = "latest"
@@ -141,6 +143,7 @@ export async function getFeed({
note: page,
useScore: true,
useHTML,
translateTo,
}),
),
)
@@ -235,6 +238,7 @@ export async function getFeed({
const expand = await expandCrossbellNote({
note: page,
useHTML,
translateTo,
})
if (expand.toNote) {
if (expand.toNote.toNote) {
@@ -251,6 +255,7 @@ export async function getFeed({
expand.toNote = await expandCrossbellNote({
note: expand.toNote,
useHTML,
translateTo,
})
delete expand.toNote.toNote
}
@@ -321,6 +326,7 @@ export async function getFeed({
useScore: true,
useStat: true,
useHTML,
translateTo,
}),
),
)
@@ -359,6 +365,7 @@ export async function getFeed({
expandCrossbellNote({
note: page,
useHTML,
translateTo,
}),
),
)
@@ -465,6 +472,7 @@ export async function getFeed({
expandCrossbellNote({
note: page,
useHTML,
translateTo,
}),
),
)
@@ -540,6 +548,7 @@ export async function getFeed({
expandCrossbellNote({
note: page,
useHTML,
translateTo,
}),
),
)
@@ -644,6 +653,7 @@ export async function getFeed({
note: page,
useHTML,
useStat: true,
translateTo,
})
return expand
},
@@ -769,6 +779,7 @@ export async function getFeed({
note: page,
useHTML,
useStat: true,
translateTo,
})
return expand
},
@@ -813,6 +824,7 @@ export async function getFeed({
useScore: false,
keyword: searchKeyword,
useHTML,
translateTo,
}),
),
)
@@ -850,6 +862,7 @@ export async function getFeed({
useStat: false,
useScore: true,
useHTML,
translateTo,
}),
),
)
+3
View File
@@ -22,6 +22,7 @@ import { checkSlugReservedWords } from "~/lib/slug-reserved-words"
import { getKeys, getStorage } from "~/lib/storage"
import {
ExpandedNote,
Language,
NoteType,
PagesSortTypes,
PageVisibilityEnum,
@@ -419,6 +420,7 @@ export async function getPage<TRender extends boolean = false>(input: {
noteId?: number
handle?: string // In order to be compatible with old drafts
disableAutofill?: boolean
translateTo?: Language
}) {
const mustLocal = input.slug?.startsWith("!local-") && !input.noteId
@@ -570,6 +572,7 @@ export async function getPage<TRender extends boolean = false>(input: {
note: page,
useStat: input.useStat,
disableAutofill: input.disableAutofill,
translateTo: input.translateTo,
})
}
+4 -206
View File
@@ -1,20 +1,14 @@
import AsyncLock from "async-lock"
import {
AnalyzeDocumentChain,
LLMChain,
loadSummarizationChain,
} from "langchain/chains"
import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains"
import { OpenAI } from "langchain/llms/openai"
import { PromptTemplate } from "langchain/prompts"
import { getLocale } from "next-intl/server"
import removeMarkdown from "remove-markdown"
import { Metadata } from "@prisma/client"
import { QueryClient } from "@tanstack/react-query"
import { defaultLocale, languageNames, locales } from "~/i18n"
import { toCid, toGateway } from "~/lib/ipfs-parser"
import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
import { defaultLocale, locales } from "~/i18n"
import { toGateway } from "~/lib/ipfs-parser"
import prisma from "~/lib/prisma.server"
import { cacheGet } from "~/lib/redis.server"
import { Language } from "~/lib/types"
@@ -38,6 +32,7 @@ export const fetchGetPage = async (
useStat: input.useStat,
noteId: input.noteId,
handle: input.handle,
translateTo: input.translateTo,
}),
}) as Promise<ReturnType<typeof pageModel.getPage>>
})
@@ -77,203 +72,6 @@ export const fetchGetPagesBySite = async (
})
}
// Content translation
type ContentTranslation = {
title?: string
content?: string
}
let translationModel4K: OpenAI | undefined
let translationModel16K: OpenAI | undefined
if (process.env.OPENAI_API_KEY) {
const options = {
openAIApiKey: process.env.OPENAI_API_KEY,
temperature: 0.2,
maxTokens: -1,
}
translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
translationModel16K = new OpenAI({
...options,
modelName: "gpt-3.5-turbo-16k",
})
}
type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
const translationChains = new Map<ChainKeyType, LLMChain>()
const getOriginalTranslation = async (
cid: string,
targetLang: Language,
originalLang?: Language,
): Promise<ContentTranslation | undefined> => {
if (!translationModel4K || !translationModel16K) return
try {
const { title, content } = await (
await fetch(toGateway(`ipfs://${cid}`))
).json()
// If the detected language is the same as the target language, return the original content
if (originalLang === targetLang) {
console.warn(
`|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
)
return
}
console.time(`fetching translation ${cid}, ${targetLang}`)
const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
includeResponse: { lang: targetLang },
})
if (!modelSize) {
console.error(
`|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
)
return
}
let chain = translationChains.get(`${modelSize}_${targetLang}`)
if (!chain) {
const prompt = new PromptTemplate({
template: `Translate the following text into "${languageNames[targetLang]}" language:
{text}
Translation:`,
inputVariables: ["text"],
})
const translateModel =
modelSize === "4k" ? translationModel4K : translationModel16K
chain = new LLMChain({ llm: translateModel, prompt })
translationChains.set(`${modelSize}_${targetLang}`, chain)
}
const t = await chain.call({ text: title })
const c = await chain.call({ text: content })
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
return {
title: t.text,
content: c.text,
}
} catch (error) {
console.error(error)
console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
}
}
async function getTranslation({
cid,
lang = "en",
}: {
cid: string
lang?: Language
}) {
const translatedContent = (await cacheGet({
key: ["translation", cid, lang],
allowEmpty: true,
noUpdate: true,
noExpire: true,
getValueFun: async () => {
let result
await lock.acquire(`translation_${cid}`, async () => {
const meta = await prisma.metadata.findFirst({
where: {
uri: `ipfs://${cid}`,
},
})
const key = "ai_translation"
const translations = meta?.[key as keyof Metadata] as Record<
string,
ContentTranslation
>
const translatedJson = translations?.[lang]
if (translatedJson) {
result = translatedJson
} else {
const newTranslation = await getOriginalTranslation(cid, lang)
if (newTranslation) {
/**
* e.g.
*
* {
* "en": {
* "title": "title",
* "content": "content"
* },
* "zh": {
* "title": "标题",
* "content": "内容"
* },
* ...
* }
*
*/
const finalTranslation = {
...translations,
[lang]: newTranslation,
}
if (meta) {
await prisma.metadata.update({
where: { uri: `ipfs://${cid}` },
data: {
[key as keyof Metadata]: finalTranslation,
},
})
} else {
await prisma.metadata.create({
data: {
uri: `ipfs://${cid}`,
[key as keyof Metadata]: finalTranslation,
},
})
}
result = newTranslation
}
}
})
return result
},
})) as ContentTranslation | undefined
return translatedContent
}
// Modify page content with translation
export async function decoratePageWithTranslation(
page?: Awaited<ReturnType<typeof pageModel.getPage>> | null,
) {
if (!page) return
const cid = toCid(page?.metadata?.uri || "")
const targetLanguage = (await getLocale()) as Language
const originalLanguage = page?.metadata?.content?.originalLanguage
if (originalLanguage === targetLanguage) {
return
}
const translatedContent = await getTranslation({
cid,
lang: targetLanguage,
})
if (translatedContent && page?.metadata?.content) {
page.metadata.content["content"] = translatedContent.content
page.metadata.content["title"] = translatedContent.title
}
}
// Post summary
let model: OpenAI | undefined