Files
xLog/src/queries/page.server.ts
T

256 lines
6.8 KiB
TypeScript

import AsyncLock from "async-lock"
import { AnalyzeDocumentChain, loadSummarizationChain } from "langchain/chains"
import { OpenAI } from "langchain/llms/openai"
import { PromptTemplate } from "langchain/prompts"
import removeMarkdown from "remove-markdown"
import { Metadata } from "@prisma/client"
import { QueryClient } from "@tanstack/react-query"
import { getNoteSlug } from "~/lib/helpers"
import { toGateway } from "~/lib/ipfs-parser"
import prisma from "~/lib/prisma.server"
import { cacheDelete, cacheGet } from "~/lib/redis.server"
import * as pageModel from "~/models/page.model"
export async function getIdBySlug(slug: string, characterId: string | number) {
slug = (slug as string)?.toLowerCase?.()
const result = (await cacheGet({
key: ["slug2id", characterId, slug],
getValueFun: async () => {
let note
let cursor = ""
do {
const response = await (
await fetch(
`https://indexer.crossbell.io/v1/notes?characterId=${characterId}&sources=xlog&cursor=${cursor}&limit=100`,
)
).json()
cursor = response.cursor
note = response?.list?.find(
(item: any) =>
slug === getNoteSlug(item) ||
slug === `${characterId}-${item.noteId}`,
)
} while (!note && cursor)
return {
noteId: note?.noteId,
}
},
noUpdate: true,
})) as {
noteId: number
}
// revalidate
if (result) {
const noteIdMatch = slug.match(`^${characterId}-(\\d+)$`)
if (!noteIdMatch?.[1]) {
fetch(
`https://indexer.crossbell.io/v1/characters/${characterId}/notes/${result.noteId}`,
)
.then((res) => res.json())
.then((note) => {
if ((note && getNoteSlug(note) !== slug) || note.deleted) {
cacheDelete(["slug2id", characterId + "", slug])
}
})
}
}
return result
}
export const fetchGetPage = async (
input: Partial<Parameters<typeof pageModel.getPage>[0]>,
queryClient: QueryClient,
) => {
const key = ["getPage", input.characterId, input]
return await queryClient.fetchQuery(key, async () => {
if (!input.characterId || !input.slug) {
return null
}
if (!input.noteId) {
const slug2Id = await getIdBySlug(input.slug, input.characterId)
if (!slug2Id?.noteId) {
return null
}
input.noteId = slug2Id.noteId
}
return cacheGet({
key,
getValueFun: () =>
pageModel.getPage({
slug: input.slug,
characterId: input.characterId!,
useStat: input.useStat,
noteId: input.noteId,
handle: input.handle,
}),
}) as Promise<ReturnType<typeof pageModel.getPage>>
})
}
export const prefetchGetPagesBySite = async (
input: Parameters<typeof pageModel.getPagesBySite>[0],
queryClient: QueryClient,
) => {
const key = ["getPagesBySite", input.characterId, input]
await queryClient.prefetchInfiniteQuery({
queryKey: key,
queryFn: async ({ pageParam }) => {
return cacheGet({
key,
getValueFun: () =>
pageModel.getPagesBySite({
...input,
cursor: pageParam,
}),
})
},
getNextPageParam: (lastPage) => lastPage.cursor || undefined,
})
}
export const fetchGetPagesBySite = async (
input: Parameters<typeof pageModel.getPagesBySite>[0],
queryClient: QueryClient,
) => {
const key = ["getPagesBySite", input.characterId, input]
return await queryClient.fetchQuery(key, async () => {
return cacheGet({
key,
getValueFun: () => pageModel.getPagesBySite(input),
}) as Promise<ReturnType<typeof pageModel.getPagesBySite>>
})
}
// Post summary
let model: OpenAI | undefined
if (process.env.OPENAI_API_KEY) {
model = new OpenAI({
openAIApiKey: process.env.OPENAI_API_KEY,
modelName: "gpt-3.5-turbo",
temperature: 0.3,
maxTokens: 400,
})
}
const chains = new Map<string, AnalyzeDocumentChain>()
const getOriginalSummary = async (cid: string, lang: string) => {
if (!model) return
try {
let { content } = await (await fetch(toGateway(`ipfs://${cid}`))).json()
if (content?.length > 5000) {
content = content.slice(0, 5000)
}
if (content?.length < 200) {
return
} else if (content) {
console.time(`fetching summary ${cid}, ${lang}`)
let chain = chains.get(lang)
if (!chain) {
const prompt = new PromptTemplate({
template: `Summarize this in "${lang}" language:
"{text}"
CONCISE SUMMARY:`,
inputVariables: ["text"],
})
const combineDocsChain = loadSummarizationChain(model, {
type: "map_reduce",
combineMapPrompt: prompt,
combinePrompt: prompt,
})
chain = new AnalyzeDocumentChain({
combineDocumentsChain: combineDocsChain,
})
chains.set(lang, chain)
}
const res = await chain.call({
input_document: removeMarkdown(content, {
useImgAltText: true,
gfm: true,
}),
})
console.timeEnd(`fetching summary ${cid}, ${lang}`)
return res?.text as string
}
} catch (error) {
console.error(error)
console.timeEnd(`fetching summary ${cid}, ${lang}`)
}
}
const lock = new AsyncLock()
export async function getSummary({
cid,
lang = "en",
}: {
cid: string
lang?: string
}) {
const summary = (await cacheGet({
key: ["summary", cid, lang],
allowEmpty: true,
noUpdate: true,
getValueFun: async () => {
if (["en", "zh", "zh-TW", "ja"].includes(lang)) {
let result
await lock.acquire(cid, async () => {
const meta = await prisma.metadata.findFirst({
where: {
uri: `ipfs://${cid}`,
},
})
const key = `ai_summary_${lang.replace("-", "").toLowerCase()}`
if (meta) {
if (meta?.[key as keyof Metadata]) {
result = meta?.[key as keyof Metadata]
} else {
const summary = await getOriginalSummary(cid, lang)
if (summary) {
await prisma.metadata.update({
where: {
uri: `ipfs://${cid}`,
},
data: {
[key as keyof Metadata]: summary,
},
})
result = summary
}
}
} else {
const summary = await getOriginalSummary(cid, lang)
if (summary) {
await prisma.metadata.create({
data: {
uri: `ipfs://${cid}`,
[key as keyof Metadata]: summary,
},
})
result = summary
}
}
})
return result
}
},
})) as string | undefined
return summary
}