From e7fa9c07814a7f7c8e78fdd24cf8e81e39375fe9 Mon Sep 17 00:00:00 2001 From: Caspian Date: Wed, 22 Nov 2023 23:59:21 +0000 Subject: [PATCH] Support to translate posts content by ai. --- package.json | 2 + pnpm-lock.yaml | 35 +++ .../migrations/20231020093655_/migration.sql | 1 + prisma/schema.prisma | 13 ++ src/app/api/feed/route.ts | 6 + src/app/site/[site]/[slug]/page.tsx | 10 +- src/lib/calculate-tokens.ts | 66 ++++++ src/lib/detect-lang.ts | 67 ++++++ src/lib/expand-unit.ts | 16 +- src/lib/i18n/settings.ts | 5 +- src/lib/llm-model-switcher-by-text-length.ts | 54 +++++ src/lib/types.ts | 3 + src/queries/page.server.ts | 209 +++++++++++++++++- 13 files changed, 478 insertions(+), 9 deletions(-) create mode 100644 prisma/migrations/20231020093655_/migration.sql create mode 100644 src/lib/calculate-tokens.ts create mode 100644 src/lib/detect-lang.ts create mode 100644 src/lib/llm-model-switcher-by-text-length.ts diff --git a/package.json b/package.json index db07db5a..363d5a7b 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "emoji-mart": "5.5.2", "fast-average-color": "9.4.0", "fast-deep-equal": "3.1.3", + "gpt-tokenizer": "^2.1.2", "hast-util-from-html": "2.0.1", "hast-util-to-html": "9.0.0", "i18next": "23.7.6", @@ -156,6 +157,7 @@ "uuid": "9.0.1", "viem": "1.19.6", "wagmi": "1.4.7", + "xregexp": "^5.1.1", "zustand": "4.4.6" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dd94c0eb..b25935ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -152,6 +152,9 @@ dependencies: fast-deep-equal: specifier: 3.1.3 version: 3.1.3 + gpt-tokenizer: + specifier: ^2.1.2 + version: 2.1.2 hast-util-from-html: specifier: 2.0.1 version: 2.0.1 @@ -383,6 +386,9 @@ dependencies: wagmi: specifier: 1.4.7 version: 1.4.7(@types/react@18.2.38)(immer@10.0.3)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.2)(viem@1.19.6)(zod@3.22.4) + xregexp: + specifier: ^5.1.1 + version: 5.1.1 zustand: specifier: 4.4.6 version: 4.4.6(@types/react@18.2.38)(immer@10.0.3)(react@18.2.0) @@ -715,6 +721,14 @@ packages: dependencies: '@babel/types': 7.23.4 + /@babel/runtime-corejs3@7.23.4: + resolution: {integrity: sha512-zQyB4MJGM+rvd4pM58n26kf3xbiitw9MHzL8oLiBMKb8MCtVDfV5nDzzJWWzLMtbvKI9wN6XwJYl479qF4JluQ==} + engines: {node: '>=6.9.0'} + dependencies: + core-js-pure: 3.33.3 + regenerator-runtime: 0.14.0 + dev: false + /@babel/runtime@7.23.4: resolution: {integrity: sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==} engines: {node: '>=6.9.0'} @@ -5181,6 +5195,11 @@ packages: toggle-selection: 1.0.6 dev: false + /core-js-pure@3.33.3: + resolution: {integrity: sha512-taJ00IDOP+XYQEA2dAe4ESkmHt1fL8wzYDo3mRWQey8uO9UojlBFMneA65kMyxfYP7106c6LzWaq7/haDT6BCQ==} + requiresBuild: true + dev: false + /core-util-is@1.0.3: resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} dev: true @@ -7079,6 +7098,12 @@ packages: dependencies: get-intrinsic: 1.2.2 + /gpt-tokenizer@2.1.2: + resolution: {integrity: sha512-HSuI5d6uey+c7x/VzQlPfCoGrfLyAc28vxWofKbjR9PJHm0AjQGSWkKw/OJnb+8S1g7nzgRsf0WH3dK+NNWYbg==} + dependencies: + rfc4648: 1.5.3 + dev: false + /graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} @@ -11078,6 +11103,10 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + /rfc4648@1.5.3: + resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==} + dev: false + /rfdc@1.3.0: resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==} dev: true @@ -12947,6 +12976,12 @@ packages: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} dev: false + /xregexp@5.1.1: + resolution: {integrity: sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==} + dependencies: + '@babel/runtime-corejs3': 7.23.4 + dev: false + /xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} diff --git a/prisma/migrations/20231020093655_/migration.sql b/prisma/migrations/20231020093655_/migration.sql new file mode 100644 index 00000000..7054d795 --- /dev/null +++ b/prisma/migrations/20231020093655_/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "Metadata" ADD COLUMN "ai_translation" JSONB; \ No newline at end of file diff --git a/prisma/schema.prisma b/prisma/schema.prisma index c47d33ff..befd1fbf 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -18,6 +18,19 @@ model Metadata { ai_summary_ja String? ai_score Int? ai_score_reason String? + // Expected JSON structure for the translation field: + // { + // "zh": { + // "title": "string", + // "content": "string", + // }, + // "jp": { + // "title": "string", + // "content": "string", + // }, + // ... + // } + ai_translation Json? @@id([uri]) @@index([uri]) } diff --git a/src/app/api/feed/route.ts b/src/app/api/feed/route.ts index b6b7c182..39542c15 100644 --- a/src/app/api/feed/route.ts +++ b/src/app/api/feed/route.ts @@ -1,5 +1,6 @@ import { getQuery, NextServerResponse } from "~/lib/server-helper" import { getFeed } from "~/models/home.model" +import { decoratePageWithTranslation } from "~/queries/page.server" export async function GET(req: Request) { const query = getQuery(req) @@ -16,6 +17,11 @@ export async function GET(req: Request) { useHTML: false, topic: query.topic, }) + + for (const item of result.list) { + await decoratePageWithTranslation(item) + } + const res = new NextServerResponse() return res.status(200).json(result) } diff --git a/src/app/site/[site]/[slug]/page.tsx b/src/app/site/[site]/[slug]/page.tsx index 0f971b44..8773e2e7 100644 --- a/src/app/site/[site]/[slug]/page.tsx +++ b/src/app/site/[site]/[slug]/page.tsx @@ -16,7 +16,11 @@ import { toCid, toGateway } from "~/lib/ipfs-parser" import { isInRN } from "~/lib/is-in-rn" import { isOnlyContent } from "~/lib/is-only-content" import getQueryClient from "~/lib/query-client" -import { fetchGetPage, getSummary } from "~/queries/page.server" +import { + decoratePageWithTranslation, + fetchGetPage, + getSummary, +} from "~/queries/page.server" import { fetchGetSite } from "~/queries/site.server" export async function generateMetadata({ @@ -40,6 +44,8 @@ export async function generateMetadata({ queryClient, ) + await decoratePageWithTranslation(page) + const title = `${ page?.metadata?.content?.title || page?.metadata?.content?.content } - ${site?.metadata?.content?.name || site?.handle}` @@ -134,6 +140,8 @@ export default async function SitePagePage({ const { i18n } = await getTranslation() const { t } = await getTranslation("common") + await decoratePageWithTranslation(page) + let summary: string | undefined if (!page.metadata.content.disableAISummary) { summary = await getSummary({ diff --git a/src/lib/calculate-tokens.ts b/src/lib/calculate-tokens.ts new file mode 100644 index 00000000..62cf242d --- /dev/null +++ b/src/lib/calculate-tokens.ts @@ -0,0 +1,66 @@ +import { encode } from "gpt-tokenizer" + +export type ModelTypes = + | "gpt-3.5-turbo" + | "gpt-3.5-turbo-0613" + | "gpt-3.5-turbo-16k" + | "gpt-3.5-turbo-16k-0613" + | "gpt-3.5-turbo-0301" + | "gpt-4" + | "gpt-4-0314" + | "gpt-4-32k-0314" + | "gpt-4-0613" + | "gpt-4-32k-0613" + +export function calculateTokens( + messages: string, + model: ModelTypes = "gpt-3.5-turbo-0613", +) { + let tokens_per_message = 0 + let tokens_per_name = 0 + if ( + [ + "gpt-3.5-turbo-0613", + "gpt-3.5-turbo-16k-0613", + "gpt-4-0314", + "gpt-4-32k-0314", + "gpt-4-0613", + "gpt-4-32k-0613", + ].includes(model) + ) { + tokens_per_message = 3 + tokens_per_name = 1 + } else if (model == "gpt-3.5-turbo-0301") { + tokens_per_message = 4 + tokens_per_name = -1 + } else if (model.includes("gpt-3.5-turbo")) { + console.log( + "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.", + ) + return calculateTokens(messages, "gpt-3.5-turbo-0613") + } else if (model.includes("gpt-4")) { + console.log( + "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.", + ) + return calculateTokens(messages, "gpt-4-0613") + } else { + throw new Error( + `num_tokens_from_messages() is not implemented for model ${model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.`, + ) + } + let num_tokens = 0 + for (let i = 0; i < messages.length; i++) { + let message = messages[i] + num_tokens += tokens_per_message + // @ts-ignore + for (let key in message) { + let value = message[key] + num_tokens += encode(value).length + if (key == "name") { + num_tokens += tokens_per_name + } + } + } + num_tokens += 3 + return num_tokens +} diff --git a/src/lib/detect-lang.ts b/src/lib/detect-lang.ts new file mode 100644 index 00000000..d5e3c927 --- /dev/null +++ b/src/lib/detect-lang.ts @@ -0,0 +1,67 @@ +import XRegExp from "xregexp" + +import { Language } from "./types" + +type Thresholds = Partial> + +const _thresholds: Thresholds = { + en: 0.8, + zh: 0.3, + ja: 0.5, +} + +export function detectLanguage( + text: string, + thresholds: Thresholds = _thresholds, +): Language { + // Pre-process the text + text = text.replace(/[\d\s\p{P}]/gu, "") // Remove numbers, whitespace, and punctuation + + // split into words + const langs: Array = text + .trim() + .split(/\s+/) + .map((word) => { + return detect(word, thresholds) + }) + + // pick the lang with the most occurrences + const result = langs.reduce<{ + k: Record + max: string | null + }>( + (acc, el) => { + if (el) { + acc.k[el] = acc.k[el] ? acc.k[el] + 1 : 1 + acc.max = acc.max ? (acc.k[acc.max] < acc.k[el] ? el : acc.max) : el + } + return acc + }, + { k: {}, max: null }, + ) + + return result.max as Language + + function detect(text: string, thresholds: Thresholds): string | null { + const scores: Record = {} + const regexes: Record = { + en: XRegExp("\\p{Latin}", "gi"), + zh: XRegExp("\\p{Han}", "gi"), + ja: XRegExp("[\\p{Hiragana}\\p{Katakana}]", "gi"), + } + + for (const [lang, regex] of Object.entries(regexes)) { + const matches = XRegExp.match(text, regex) || [] + const score = matches.length / text.length + if (score && score >= thresholds[lang as keyof Thresholds]!) { + return lang + } + scores[lang] = score + } + + if (Object.keys(scores).length === 0) return null + + // pick lang with highest percentage + return Object.keys(scores).reduce((a, b) => (scores[a] > scores[b] ? a : b)) + } +} diff --git a/src/lib/expand-unit.ts b/src/lib/expand-unit.ts index 1c8b5c40..ae0e3584 100644 --- a/src/lib/expand-unit.ts +++ b/src/lib/expand-unit.ts @@ -7,6 +7,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser" import readingTime from "~/lib/reading-time" import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types" +import { detectLanguage } from "./detect-lang" import { getNoteSlug } from "./helpers" export const expandCrossbellNote = async ({ @@ -49,12 +50,11 @@ export const expandCrossbellNote = async ({ let rendered if (expandedNote.metadata?.content?.content) { const { renderPageContent } = await import("~/markdown") - rendered = renderPageContent(expandedNote.metadata.content.content, true) + const content = expandedNote.metadata.content.content + rendered = renderPageContent(content, true) if (keyword) { - const position = expandedNote.metadata.content.content - .toLowerCase() - .indexOf(keyword.toLowerCase()) - expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content.slice( + const position = content.toLowerCase().indexOf(keyword.toLowerCase()) + expandedNote.metadata.content.summary = `...${content.slice( position - 10, position + 100, )}` @@ -64,6 +64,12 @@ export const expandCrossbellNote = async ({ } } + const processedContent = removeMarkdown( + content.replace(/```[^]+?```/g, ""), + ) + const detectedLang = detectLanguage(processedContent) + expandedNote.metadata.content.originalLanguage = detectedLang + expandedNote.metadata.content.audio = rendered.audio expandedNote.metadata.content.frontMatter = rendered.frontMatter diff --git a/src/lib/i18n/settings.ts b/src/lib/i18n/settings.ts index 7b70c27a..31baae47 100644 --- a/src/lib/i18n/settings.ts +++ b/src/lib/i18n/settings.ts @@ -1,5 +1,8 @@ +import { Language } from "../types" + export const fallbackLng = "en" -export const languages = ["en", "zh", "zh-TW", "ja"] +export const languages: Language[] = ["en", "zh", "zh-TW", "ja"] + export const languageNames = { en: "English", zh: "中文", diff --git a/src/lib/llm-model-switcher-by-text-length.ts b/src/lib/llm-model-switcher-by-text-length.ts new file mode 100644 index 00000000..6cef8b35 --- /dev/null +++ b/src/lib/llm-model-switcher-by-text-length.ts @@ -0,0 +1,54 @@ +import { calculateTokens } from "./calculate-tokens" +import { Language } from "./types" + +function getTolerantCount(textLength: "4k" | "16k"): number { + const ratio = 0.8 + + if (textLength == "4k") { + return 4000 * ratio + } + + return 16000 +} + +// TODO: This is a temporary solution. Because we don't know the language of the original text, we can't calculate the tokens accurately. Currently, only "Chinese to other languages" is supported. +export function llmModelSwitcherByTextLength( + text: string, + options: { + // If true, include the response in the text length calculation with the specified language + includeResponse?: { lang: Language } + } = {}, +): { + modelSize: "4k" | "16k" | undefined + tokens: number +} { + const { includeResponse } = options + const tokens = calculateTokens(text) + + const langRatioMapping: { [key in Language]: number } = { + en: 1.6, + ja: 1.25, + zh: 1, + "zh-TW": 1, + } + + const finalCount = + tokens / (includeResponse ? langRatioMapping[includeResponse.lang] : 1) + + if (finalCount < getTolerantCount("4k")) { + return { + modelSize: "4k", + tokens, + } + } else if (finalCount < getTolerantCount("16k")) { + return { + modelSize: "16k", + tokens, + } + } + + return { + modelSize: undefined, + tokens, + } +} diff --git a/src/lib/types.ts b/src/lib/types.ts index 1b190844..b970bde6 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -1,5 +1,7 @@ import type { CharacterEntity, NoteEntity } from "crossbell" +export type Language = "en" | "zh" | "zh-TW" | "ja" + export type Site = { id: string name: string @@ -89,6 +91,7 @@ export type ExpandedNote = NoteEntity & { summary?: string cover?: string images?: string[] + originalLanguage?: Language frontMatter?: Record slug?: string audio?: string diff --git a/src/queries/page.server.ts b/src/queries/page.server.ts index b0958619..6759122d 100644 --- a/src/queries/page.server.ts +++ b/src/queries/page.server.ts @@ -1,5 +1,9 @@ import AsyncLock from "async-lock" -import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains" +import { + AnalyzeDocumentChain, + LLMChain, + loadSummarizationChain, +} from "langchain/chains" import { OpenAI } from "langchain/llms/openai" import { PromptTemplate } from "langchain/prompts" import removeMarkdown from "remove-markdown" @@ -7,9 +11,13 @@ import removeMarkdown from "remove-markdown" import { Metadata } from "@prisma/client" import { QueryClient } from "@tanstack/react-query" -import { toGateway } from "~/lib/ipfs-parser" +import { getTranslation as getTranslationWithI18n } from "~/lib/i18n" +import { languageNames } from "~/lib/i18n/settings" +import { toCid, toGateway } from "~/lib/ipfs-parser" +import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length" import prisma from "~/lib/prisma.server" import { cacheGet } from "~/lib/redis.server" +import { Language } from "~/lib/types" import * as pageModel from "~/models/page.model" export const fetchGetPage = async ( @@ -69,6 +77,203 @@ export const fetchGetPagesBySite = async ( }) } +// Content translation + +type ContentTranslation = { + title?: string + content?: string +} + +let translationModel4K: OpenAI | undefined +let translationModel16K: OpenAI | undefined +if (process.env.OPENAI_API_KEY) { + const options = { + openAIApiKey: process.env.OPENAI_API_KEY, + temperature: 0.2, + maxTokens: -1, + } + translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" }) + translationModel16K = new OpenAI({ + ...options, + modelName: "gpt-3.5-turbo-16k", + }) +} + +type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja" +const translationChains = new Map() + +const getOriginalTranslation = async ( + cid: string, + targetLang: Language, + originalLang?: Language, +): Promise => { + if (!translationModel4K || !translationModel16K) return + + try { + const { title, content } = await ( + await fetch(toGateway(`ipfs://${cid}`)) + ).json() + + // If the detected language is the same as the target language, return the original content + if (originalLang === targetLang) { + console.warn( + `|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`, + ) + return + } + + console.time(`fetching translation ${cid}, ${targetLang}`) + const { modelSize, tokens } = llmModelSwitcherByTextLength(content, { + includeResponse: { lang: targetLang }, + }) + + if (!modelSize) { + console.error( + `|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`, + ) + return + } + + let chain = translationChains.get(`${modelSize}_${targetLang}`) + + if (!chain) { + const prompt = new PromptTemplate({ + template: `Translate the following text into "${languageNames[targetLang]}" language: + {text} + Translation:`, + inputVariables: ["text"], + }) + + const translateModel = + modelSize === "4k" ? translationModel4K : translationModel16K + + chain = new LLMChain({ llm: translateModel, prompt }) + + translationChains.set(`${modelSize}_${targetLang}`, chain) + } + + const t = await chain.call({ text: title }) + const c = await chain.call({ text: content }) + + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + + return { + title: t.text, + content: c.text, + } + } catch (error) { + console.error(error) + console.timeEnd(`fetching translation ${cid}, ${targetLang}`) + } +} + +async function getTranslation({ + cid, + lang = "en", +}: { + cid: string + lang?: Language +}) { + const translatedContent = (await cacheGet({ + key: ["translation", cid, lang], + allowEmpty: true, + noUpdate: true, + noExpire: true, + getValueFun: async () => { + let result + + await lock.acquire(`translation_${cid}`, async () => { + const meta = await prisma.metadata.findFirst({ + where: { + uri: `ipfs://${cid}`, + }, + }) + + const key = "ai_translation" + const translations = meta?.[key as keyof Metadata] as Record< + string, + ContentTranslation + > + const translatedJson = translations?.[lang] + + if (translatedJson) { + result = translatedJson + } else { + const newTranslation = await getOriginalTranslation(cid, lang) + if (newTranslation) { + /** + * e.g. + * + * { + * "en": { + * "title": "title", + * "content": "content" + * }, + * "zh": { + * "title": "标题", + * "content": "内容" + * }, + * ... + * } + * + */ + const finalTranslation = { + ...translations, + [lang]: newTranslation, + } + + if (meta) { + await prisma.metadata.update({ + where: { uri: `ipfs://${cid}` }, + data: { + [key as keyof Metadata]: finalTranslation, + }, + }) + } else { + await prisma.metadata.create({ + data: { + uri: `ipfs://${cid}`, + [key as keyof Metadata]: finalTranslation, + }, + }) + } + result = newTranslation + } + } + }) + + return result + }, + })) as ContentTranslation | undefined + + return translatedContent +} + +// Modify page content with translation +export async function decoratePageWithTranslation( + page?: Awaited> | null, +) { + if (!page) return + const cid = toCid(page?.metadata?.uri || "") + const { i18n } = await getTranslationWithI18n() + const targetLanguage = i18n.resolvedLanguage as Language + const originalLanguage = page?.metadata?.content?.originalLanguage + + if (originalLanguage === targetLanguage) { + return + } + + const translatedContent = await getTranslation({ + cid, + lang: targetLanguage, + }) + + if (translatedContent && page?.metadata?.content) { + page.metadata.content["content"] = translatedContent.content + page.metadata.content["title"] = translatedContent.title + } +} + // Post summary let model: OpenAI | undefined