Support to translate posts content by ai.

package.json

@@ -79,6 +79,7 @@
     "emoji-mart": "5.5.2",
     "fast-average-color": "9.4.0",
     "fast-deep-equal": "3.1.3",
+    "gpt-tokenizer": "^2.1.2",
     "hast-util-from-html": "2.0.1",
     "hast-util-to-html": "9.0.0",
     "i18next": "23.7.6",
@@ -156,6 +157,7 @@
     "uuid": "9.0.1",
     "viem": "1.19.6",
     "wagmi": "1.4.7",
+    "xregexp": "^5.1.1",
     "zustand": "4.4.6"
   },
   "devDependencies": {

pnpm-lock.yaml

@@ -152,6 +152,9 @@ dependencies:
   fast-deep-equal:
     specifier: 3.1.3
     version: 3.1.3
+  gpt-tokenizer:
+    specifier: ^2.1.2
+    version: 2.1.2
   hast-util-from-html:
     specifier: 2.0.1
     version: 2.0.1
@@ -383,6 +386,9 @@ dependencies:
   wagmi:
     specifier: 1.4.7
     version: 1.4.7(@types/react@18.2.38)(immer@10.0.3)(react-dom@18.2.0)(react@18.2.0)(typescript@5.3.2)(viem@1.19.6)(zod@3.22.4)
+  xregexp:
+    specifier: ^5.1.1
+    version: 5.1.1
   zustand:
     specifier: 4.4.6
     version: 4.4.6(@types/react@18.2.38)(immer@10.0.3)(react@18.2.0)
@@ -715,6 +721,14 @@ packages:
     dependencies:
       '@babel/types': 7.23.4
 
+  /@babel/runtime-corejs3@7.23.4:
+    resolution: {integrity: sha512-zQyB4MJGM+rvd4pM58n26kf3xbiitw9MHzL8oLiBMKb8MCtVDfV5nDzzJWWzLMtbvKI9wN6XwJYl479qF4JluQ==}
+    engines: {node: '>=6.9.0'}
+    dependencies:
+      core-js-pure: 3.33.3
+      regenerator-runtime: 0.14.0
+    dev: false
+
   /@babel/runtime@7.23.4:
     resolution: {integrity: sha512-2Yv65nlWnWlSpe3fXEyX5i7fx5kIKo4Qbcj+hMO0odwaneFjfXw5fdum+4yL20O0QiaHpia0cYQ9xpNMqrBwHg==}
     engines: {node: '>=6.9.0'}
@@ -5181,6 +5195,11 @@ packages:
       toggle-selection: 1.0.6
     dev: false
 
+  /core-js-pure@3.33.3:
+    resolution: {integrity: sha512-taJ00IDOP+XYQEA2dAe4ESkmHt1fL8wzYDo3mRWQey8uO9UojlBFMneA65kMyxfYP7106c6LzWaq7/haDT6BCQ==}
+    requiresBuild: true
+    dev: false
+
   /core-util-is@1.0.3:
     resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==}
     dev: true
@@ -7079,6 +7098,12 @@ packages:
     dependencies:
       get-intrinsic: 1.2.2
 
+  /gpt-tokenizer@2.1.2:
+    resolution: {integrity: sha512-HSuI5d6uey+c7x/VzQlPfCoGrfLyAc28vxWofKbjR9PJHm0AjQGSWkKw/OJnb+8S1g7nzgRsf0WH3dK+NNWYbg==}
+    dependencies:
+      rfc4648: 1.5.3
+    dev: false
+
   /graceful-fs@4.2.11:
     resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==}
 
@@ -11078,6 +11103,10 @@ packages:
     resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==}
     engines: {iojs: '>=1.0.0', node: '>=0.10.0'}
 
+  /rfc4648@1.5.3:
+    resolution: {integrity: sha512-MjOWxM065+WswwnmNONOT+bD1nXzY9Km6u3kzvnx8F8/HXGZdz3T6e6vZJ8Q/RIMUSp/nxqjH3GwvJDy8ijeQQ==}
+    dev: false
+
   /rfdc@1.3.0:
     resolution: {integrity: sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==}
     dev: true
@@ -12947,6 +12976,12 @@ packages:
     resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
     dev: false
 
+  /xregexp@5.1.1:
+    resolution: {integrity: sha512-fKXeVorD+CzWvFs7VBuKTYIW63YD1e1osxwQ8caZ6o1jg6pDAbABDG54LCIq0j5cy7PjRvGIq6sef9DYPXpncg==}
+    dependencies:
+      '@babel/runtime-corejs3': 7.23.4
+    dev: false
+
   /xtend@4.0.2:
     resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
     engines: {node: '>=0.4'}

prisma/migrations/20231020093655_/migration.sql

@@ -0,0 +1 @@
+ALTER TABLE "Metadata" ADD COLUMN "ai_translation" JSONB;

prisma/schema.prisma

@@ -18,6 +18,19 @@ model Metadata {
   ai_summary_ja   String?
   ai_score     Int?
   ai_score_reason String?
+  // Expected JSON structure for the translation field:
+  // {
+  //   "zh": {
+  //     "title": "string",
+  //     "content": "string",
+  //   },
+  //   "jp": {
+  //     "title": "string",
+  //     "content": "string",
+  //   },
+  //   ...
+  // } 
+  ai_translation   Json?
   @@id([uri])
   @@index([uri])
 }

src/app/api/feed/route.ts

@@ -1,5 +1,6 @@
 import { getQuery, NextServerResponse } from "~/lib/server-helper"
 import { getFeed } from "~/models/home.model"
+import { decoratePageWithTranslation } from "~/queries/page.server"
 
 export async function GET(req: Request) {
   const query = getQuery(req)
@@ -16,6 +17,11 @@ export async function GET(req: Request) {
     useHTML: false,
     topic: query.topic,
   })
+
+  for (const item of result.list) {
+    await decoratePageWithTranslation(item)
+  }
+
   const res = new NextServerResponse()
   return res.status(200).json(result)
 }

src/app/site/[site]/[slug]/page.tsx

@@ -16,7 +16,11 @@ import { toCid, toGateway } from "~/lib/ipfs-parser"
 import { isInRN } from "~/lib/is-in-rn"
 import { isOnlyContent } from "~/lib/is-only-content"
 import getQueryClient from "~/lib/query-client"
-import { fetchGetPage, getSummary } from "~/queries/page.server"
+import {
+  decoratePageWithTranslation,
+  fetchGetPage,
+  getSummary,
+} from "~/queries/page.server"
 import { fetchGetSite } from "~/queries/site.server"
 
 export async function generateMetadata({
@@ -40,6 +44,8 @@ export async function generateMetadata({
     queryClient,
   )
 
+  await decoratePageWithTranslation(page)
+
   const title = `${
     page?.metadata?.content?.title || page?.metadata?.content?.content
   } - ${site?.metadata?.content?.name || site?.handle}`
@@ -134,6 +140,8 @@ export default async function SitePagePage({
 
   const { i18n } = await getTranslation()
   const { t } = await getTranslation("common")
+  await decoratePageWithTranslation(page)
+
   let summary: string | undefined
   if (!page.metadata.content.disableAISummary) {
     summary = await getSummary({

src/lib/calculate-tokens.ts

@@ -0,0 +1,66 @@
+import { encode } from "gpt-tokenizer"
+
+export type ModelTypes =
+  | "gpt-3.5-turbo"
+  | "gpt-3.5-turbo-0613"
+  | "gpt-3.5-turbo-16k"
+  | "gpt-3.5-turbo-16k-0613"
+  | "gpt-3.5-turbo-0301"
+  | "gpt-4"
+  | "gpt-4-0314"
+  | "gpt-4-32k-0314"
+  | "gpt-4-0613"
+  | "gpt-4-32k-0613"
+
+export function calculateTokens(
+  messages: string,
+  model: ModelTypes = "gpt-3.5-turbo-0613",
+) {
+  let tokens_per_message = 0
+  let tokens_per_name = 0
+  if (
+    [
+      "gpt-3.5-turbo-0613",
+      "gpt-3.5-turbo-16k-0613",
+      "gpt-4-0314",
+      "gpt-4-32k-0314",
+      "gpt-4-0613",
+      "gpt-4-32k-0613",
+    ].includes(model)
+  ) {
+    tokens_per_message = 3
+    tokens_per_name = 1
+  } else if (model == "gpt-3.5-turbo-0301") {
+    tokens_per_message = 4
+    tokens_per_name = -1
+  } else if (model.includes("gpt-3.5-turbo")) {
+    console.log(
+      "Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.",
+    )
+    return calculateTokens(messages, "gpt-3.5-turbo-0613")
+  } else if (model.includes("gpt-4")) {
+    console.log(
+      "Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.",
+    )
+    return calculateTokens(messages, "gpt-4-0613")
+  } else {
+    throw new Error(
+      `num_tokens_from_messages() is not implemented for model ${model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.`,
+    )
+  }
+  let num_tokens = 0
+  for (let i = 0; i < messages.length; i++) {
+    let message = messages[i]
+    num_tokens += tokens_per_message
+    // @ts-ignore
+    for (let key in message) {
+      let value = message[key]
+      num_tokens += encode(value).length
+      if (key == "name") {
+        num_tokens += tokens_per_name
+      }
+    }
+  }
+  num_tokens += 3
+  return num_tokens
+}

src/lib/detect-lang.ts

@@ -0,0 +1,67 @@
+import XRegExp from "xregexp"
+
+import { Language } from "./types"
+
+type Thresholds = Partial<Record<Language, number>>
+
+const _thresholds: Thresholds = {
+  en: 0.8,
+  zh: 0.3,
+  ja: 0.5,
+}
+
+export function detectLanguage(
+  text: string,
+  thresholds: Thresholds = _thresholds,
+): Language {
+  // Pre-process the text
+  text = text.replace(/[\d\s\p{P}]/gu, "") // Remove numbers, whitespace, and punctuation
+
+  // split into words
+  const langs: Array<string | null> = text
+    .trim()
+    .split(/\s+/)
+    .map((word) => {
+      return detect(word, thresholds)
+    })
+
+  // pick the lang with the most occurrences
+  const result = langs.reduce<{
+    k: Record<string, number>
+    max: string | null
+  }>(
+    (acc, el) => {
+      if (el) {
+        acc.k[el] = acc.k[el] ? acc.k[el] + 1 : 1
+        acc.max = acc.max ? (acc.k[acc.max] < acc.k[el] ? el : acc.max) : el
+      }
+      return acc
+    },
+    { k: {}, max: null },
+  )
+
+  return result.max as Language
+
+  function detect(text: string, thresholds: Thresholds): string | null {
+    const scores: Record<string, number> = {}
+    const regexes: Record<string, RegExp> = {
+      en: XRegExp("\\p{Latin}", "gi"),
+      zh: XRegExp("\\p{Han}", "gi"),
+      ja: XRegExp("[\\p{Hiragana}\\p{Katakana}]", "gi"),
+    }
+
+    for (const [lang, regex] of Object.entries(regexes)) {
+      const matches = XRegExp.match(text, regex) || []
+      const score = matches.length / text.length
+      if (score && score >= thresholds[lang as keyof Thresholds]!) {
+        return lang
+      }
+      scores[lang] = score
+    }
+
+    if (Object.keys(scores).length === 0) return null
+
+    // pick lang with highest percentage
+    return Object.keys(scores).reduce((a, b) => (scores[a] > scores[b] ? a : b))
+  }
+}

src/lib/expand-unit.ts

@@ -7,6 +7,7 @@ import { toCid, toGateway } from "~/lib/ipfs-parser"
 import readingTime from "~/lib/reading-time"
 import { ExpandedCharacter, ExpandedNote, PortfolioStats } from "~/lib/types"
 
+import { detectLanguage } from "./detect-lang"
 import { getNoteSlug } from "./helpers"
 
 export const expandCrossbellNote = async ({
@@ -49,12 +50,11 @@ export const expandCrossbellNote = async ({
     let rendered
     if (expandedNote.metadata?.content?.content) {
       const { renderPageContent } = await import("~/markdown")
-      rendered = renderPageContent(expandedNote.metadata.content.content, true)
+      const content = expandedNote.metadata.content.content
+      rendered = renderPageContent(content, true)
       if (keyword) {
-        const position = expandedNote.metadata.content.content
-          .toLowerCase()
-          .indexOf(keyword.toLowerCase())
-        expandedNote.metadata.content.summary = `...${expandedNote.metadata.content.content.slice(
+        const position = content.toLowerCase().indexOf(keyword.toLowerCase())
+        expandedNote.metadata.content.summary = `...${content.slice(
           position - 10,
           position + 100,
         )}`
@@ -64,6 +64,12 @@ export const expandCrossbellNote = async ({
         }
       }
 
+      const processedContent = removeMarkdown(
+        content.replace(/```[^]+?```/g, ""),
+      )
+      const detectedLang = detectLanguage(processedContent)
+      expandedNote.metadata.content.originalLanguage = detectedLang
+
       expandedNote.metadata.content.audio = rendered.audio
       expandedNote.metadata.content.frontMatter = rendered.frontMatter
 

src/lib/i18n/settings.ts

@@ -1,5 +1,8 @@
+import { Language } from "../types"
+
 export const fallbackLng = "en"
-export const languages = ["en", "zh", "zh-TW", "ja"]
+export const languages: Language[] = ["en", "zh", "zh-TW", "ja"]
+
 export const languageNames = {
   en: "English",
   zh: "中文",

src/lib/llm-model-switcher-by-text-length.ts

@@ -0,0 +1,54 @@
+import { calculateTokens } from "./calculate-tokens"
+import { Language } from "./types"
+
+function getTolerantCount(textLength: "4k" | "16k"): number {
+  const ratio = 0.8
+
+  if (textLength == "4k") {
+    return 4000 * ratio
+  }
+
+  return 16000
+}
+
+// TODO: This is a temporary solution. Because we don't know the language of the original text, we can't calculate the tokens accurately. Currently, only "Chinese to other languages" is supported.
+export function llmModelSwitcherByTextLength(
+  text: string,
+  options: {
+    // If true, include the response in the text length calculation with the specified language
+    includeResponse?: { lang: Language }
+  } = {},
+): {
+  modelSize: "4k" | "16k" | undefined
+  tokens: number
+} {
+  const { includeResponse } = options
+  const tokens = calculateTokens(text)
+
+  const langRatioMapping: { [key in Language]: number } = {
+    en: 1.6,
+    ja: 1.25,
+    zh: 1,
+    "zh-TW": 1,
+  }
+
+  const finalCount =
+    tokens / (includeResponse ? langRatioMapping[includeResponse.lang] : 1)
+
+  if (finalCount < getTolerantCount("4k")) {
+    return {
+      modelSize: "4k",
+      tokens,
+    }
+  } else if (finalCount < getTolerantCount("16k")) {
+    return {
+      modelSize: "16k",
+      tokens,
+    }
+  }
+
+  return {
+    modelSize: undefined,
+    tokens,
+  }
+}

src/lib/types.ts

@@ -1,5 +1,7 @@
 import type { CharacterEntity, NoteEntity } from "crossbell"
 
+export type Language = "en" | "zh" | "zh-TW" | "ja"
+
 export type Site = {
   id: string
   name: string
@@ -89,6 +91,7 @@ export type ExpandedNote = NoteEntity & {
       summary?: string
       cover?: string
       images?: string[]
+      originalLanguage?: Language
       frontMatter?: Record<string, any>
       slug?: string
       audio?: string

src/queries/page.server.ts

@@ -1,5 +1,9 @@
 import AsyncLock from "async-lock"
-import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains"
+import {
+  AnalyzeDocumentChain,
+  LLMChain,
+  loadSummarizationChain,
+} from "langchain/chains"
 import { OpenAI } from "langchain/llms/openai"
 import { PromptTemplate } from "langchain/prompts"
 import removeMarkdown from "remove-markdown"
@@ -7,9 +11,13 @@ import removeMarkdown from "remove-markdown"
 import { Metadata } from "@prisma/client"
 import { QueryClient } from "@tanstack/react-query"
 
-import { toGateway } from "~/lib/ipfs-parser"
+import { getTranslation as getTranslationWithI18n } from "~/lib/i18n"
+import { languageNames } from "~/lib/i18n/settings"
+import { toCid, toGateway } from "~/lib/ipfs-parser"
+import { llmModelSwitcherByTextLength } from "~/lib/llm-model-switcher-by-text-length"
 import prisma from "~/lib/prisma.server"
 import { cacheGet } from "~/lib/redis.server"
+import { Language } from "~/lib/types"
 import * as pageModel from "~/models/page.model"
 
 export const fetchGetPage = async (
@@ -69,6 +77,203 @@ export const fetchGetPagesBySite = async (
   })
 }
 
+// Content translation
+
+type ContentTranslation = {
+  title?: string
+  content?: string
+}
+
+let translationModel4K: OpenAI | undefined
+let translationModel16K: OpenAI | undefined
+if (process.env.OPENAI_API_KEY) {
+  const options = {
+    openAIApiKey: process.env.OPENAI_API_KEY,
+    temperature: 0.2,
+    maxTokens: -1,
+  }
+  translationModel4K = new OpenAI({ ...options, modelName: "gpt-3.5-turbo" })
+  translationModel16K = new OpenAI({
+    ...options,
+    modelName: "gpt-3.5-turbo-16k",
+  })
+}
+
+type ChainKeyType = `${4 | 16}k_${Language}` // e.g. "4k_en" | "4k_zh" | "4k_zh-TW" | "4k_ja" | "16k_en" | "16k_zh" | "16k_zh-TW" | "16k_ja"
+const translationChains = new Map<ChainKeyType, LLMChain>()
+
+const getOriginalTranslation = async (
+  cid: string,
+  targetLang: Language,
+  originalLang?: Language,
+): Promise<ContentTranslation | undefined> => {
+  if (!translationModel4K || !translationModel16K) return
+
+  try {
+    const { title, content } = await (
+      await fetch(toGateway(`ipfs://${cid}`))
+    ).json()
+
+    // If the detected language is the same as the target language, return the original content
+    if (originalLang === targetLang) {
+      console.warn(
+        `|__ Warn: Detected language is the same as the target language, return the original content: ${cid}, ${targetLang}`,
+      )
+      return
+    }
+
+    console.time(`fetching translation ${cid}, ${targetLang}`)
+    const { modelSize, tokens } = llmModelSwitcherByTextLength(content, {
+      includeResponse: { lang: targetLang },
+    })
+
+    if (!modelSize) {
+      console.error(
+        `|__ Error: Content too long for translation: ${cid}, ${targetLang}. (Tokens: ${tokens})`,
+      )
+      return
+    }
+
+    let chain = translationChains.get(`${modelSize}_${targetLang}`)
+
+    if (!chain) {
+      const prompt = new PromptTemplate({
+        template: `Translate the following text into "${languageNames[targetLang]}" language: 
+        {text}
+        Translation:`,
+        inputVariables: ["text"],
+      })
+
+      const translateModel =
+        modelSize === "4k" ? translationModel4K : translationModel16K
+
+      chain = new LLMChain({ llm: translateModel, prompt })
+
+      translationChains.set(`${modelSize}_${targetLang}`, chain)
+    }
+
+    const t = await chain.call({ text: title })
+    const c = await chain.call({ text: content })
+
+    console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
+
+    return {
+      title: t.text,
+      content: c.text,
+    }
+  } catch (error) {
+    console.error(error)
+    console.timeEnd(`fetching translation ${cid}, ${targetLang}`)
+  }
+}
+
+async function getTranslation({
+  cid,
+  lang = "en",
+}: {
+  cid: string
+  lang?: Language
+}) {
+  const translatedContent = (await cacheGet({
+    key: ["translation", cid, lang],
+    allowEmpty: true,
+    noUpdate: true,
+    noExpire: true,
+    getValueFun: async () => {
+      let result
+
+      await lock.acquire(`translation_${cid}`, async () => {
+        const meta = await prisma.metadata.findFirst({
+          where: {
+            uri: `ipfs://${cid}`,
+          },
+        })
+
+        const key = "ai_translation"
+        const translations = meta?.[key as keyof Metadata] as Record<
+          string,
+          ContentTranslation
+        >
+        const translatedJson = translations?.[lang]
+
+        if (translatedJson) {
+          result = translatedJson
+        } else {
+          const newTranslation = await getOriginalTranslation(cid, lang)
+          if (newTranslation) {
+            /**
+             * e.g.
+             *
+             * {
+             *  "en": {
+             *    "title": "title",
+             *    "content": "content"
+             *  },
+             *  "zh": {
+             *    "title": "标题",
+             *    "content": "内容"
+             *  },
+             *  ...
+             * }
+             *
+             */
+            const finalTranslation = {
+              ...translations,
+              [lang]: newTranslation,
+            }
+
+            if (meta) {
+              await prisma.metadata.update({
+                where: { uri: `ipfs://${cid}` },
+                data: {
+                  [key as keyof Metadata]: finalTranslation,
+                },
+              })
+            } else {
+              await prisma.metadata.create({
+                data: {
+                  uri: `ipfs://${cid}`,
+                  [key as keyof Metadata]: finalTranslation,
+                },
+              })
+            }
+            result = newTranslation
+          }
+        }
+      })
+
+      return result
+    },
+  })) as ContentTranslation | undefined
+
+  return translatedContent
+}
+
+// Modify page content with translation
+export async function decoratePageWithTranslation(
+  page?: Awaited<ReturnType<typeof pageModel.getPage>> | null,
+) {
+  if (!page) return
+  const cid = toCid(page?.metadata?.uri || "")
+  const { i18n } = await getTranslationWithI18n()
+  const targetLanguage = i18n.resolvedLanguage as Language
+  const originalLanguage = page?.metadata?.content?.originalLanguage
+
+  if (originalLanguage === targetLanguage) {
+    return
+  }
+
+  const translatedContent = await getTranslation({
+    cid,
+    lang: targetLanguage,
+  })
+
+  if (translatedContent && page?.metadata?.content) {
+    page.metadata.content["content"] = translatedContent.content
+    page.metadata.content["title"] = translatedContent.title
+  }
+}
+
 // Post summary
 
 let model: OpenAI | undefined