From ef2f414df39fc0c7f51f746f4f831aaf85ba7149 Mon Sep 17 00:00:00 2001 From: DIYgod Date: Thu, 30 Mar 2023 16:55:12 +0100 Subject: [PATCH] feat: ai filtering for latest feed --- .../migrations/20230329235052_/migration.sql | 2 + prisma/schema.prisma | 1 + src/components/main/MainFeed.tsx | 257 +++++++++++------- src/lib/ipfs-parser.ts | 1 + src/models/page.model.ts | 4 + src/pages/api/score.ts | 121 +++++++++ src/pages/api/summary.ts | 56 ++-- src/queries/page.ts | 11 + 8 files changed, 322 insertions(+), 131 deletions(-) create mode 100644 prisma/migrations/20230329235052_/migration.sql create mode 100644 src/pages/api/score.ts diff --git a/prisma/migrations/20230329235052_/migration.sql b/prisma/migrations/20230329235052_/migration.sql new file mode 100644 index 00000000..e81b86ab --- /dev/null +++ b/prisma/migrations/20230329235052_/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "Metadata" ADD COLUMN "ai_score_reason" TEXT; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 7d18767d..c47d33ff 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -17,6 +17,7 @@ model Metadata { ai_summary_zhtw String? ai_summary_ja String? ai_score Int? + ai_score_reason String? @@id([uri]) @@index([uri]) } diff --git a/src/components/main/MainFeed.tsx b/src/components/main/MainFeed.tsx index a1d53c9e..3f657836 100644 --- a/src/components/main/MainFeed.tsx +++ b/src/components/main/MainFeed.tsx @@ -3,13 +3,130 @@ import { useDate } from "~/hooks/useDate" import { EmptyState } from "../ui/EmptyState" import { useRouter } from "next/router" import { Image } from "~/components/ui/Image" -import { Button } from "~/components/ui/Button" import { useTranslation } from "next-i18next" -import { useEffect, useState } from "react" import { useGetFeed } from "~/queries/home" import { CharacterFloatCard } from "~/components/common/CharacterFloatCard" import { useAccountState } from "@crossbell/connect-kit" import InfiniteScroll from "react-infinite-scroller" +import { ExpandedNote } from "~/lib/types" +import { useGetScore } from "~/queries/page" +import { toCid } from "~/lib/ipfs-parser" +import { cn } from "~/lib/utils" +import { useEffect, useState } from "react" +import { Switch } from "@headlessui/react" +import { setStorage, getStorage } from "~/lib/storage" + +const Post = ({ + post, + filtering, +}: { + post: ExpandedNote + filtering: number +}) => { + const router = useRouter() + const { t } = useTranslation(["common", "site"]) + const date = useDate() + const score = useGetScore({ + cid: toCid(post.metadata.uri || ""), + }) + + return ( +
+
+ +
+ + {post.character?.handle + + + {post.character?.metadata?.content?.name || + post.character?.handle} + +
+ + · + +
+ +
+

+ {post.metadata?.content?.title} +

+
+ {!!post.metadata?.content?.tags?.filter( + (tag) => tag !== "post" && tag !== "page", + ).length && ( + + {post.metadata?.content?.tags + ?.filter((tag) => tag !== "post" && tag !== "page") + .map((tag) => ( + { + e.preventDefault() + router.push(`/tag/${tag}`) + }} + > + #{tag} + + ))} + + )} +
+
+ {post.metadata?.content?.summary} + {post.metadata?.content?.summary && "..."} +
+
+ {post.metadata?.content.cover && ( +
+ cover +
+ )} + +
+ ) +} export const MainFeed: React.FC<{ type?: "latest" | "recommend" | "following" @@ -23,18 +140,14 @@ export const MainFeed: React.FC<{ characterId: currentCharacterId, }) - const router = useRouter() - const { t } = useTranslation(["common", "site"]) - const date = useDate() + const hasFiltering = type === "latest" - const [isMounted, setIsMounted] = useState(false) + const [aiFiltering, setAiFiltering] = useState(false) useEffect(() => { - setIsMounted(true) + setAiFiltering(getStorage("ai_filtering")?.enabled || false) }, []) - let currentLength = 0 - return ( <> } > + {hasFiltering && ( +
+ + Enable AI Filtering + { + setAiFiltering(value) + setStorage("ai_filtering", { + enabled: value, + }) + }} + className={`${ + aiFiltering ? "bg-accent" : "bg-gray-200" + } ml-5 relative inline-flex h-6 w-11 items-center rounded-full`} + > + Enable AI Filtering + + +
+ )} {feed.isLoading ? (
Loading...
) : !feed.data?.pages[0]?.count ? ( @@ -57,103 +195,12 @@ export const MainFeed: React.FC<{
{feed.data?.pages.map((posts) => posts?.list.map((post) => { - currentLength++ return ( -
-
- -
- - {post.character?.handle - - - {post.character?.metadata?.content?.name || - post.character?.handle} - -
- - · - -
- -
-

- {post.metadata?.content?.title} -

-
- {!!post.metadata?.content?.tags?.filter( - (tag) => tag !== "post" && tag !== "page", - ).length && ( - - {post.metadata?.content?.tags - ?.filter( - (tag) => tag !== "post" && tag !== "page", - ) - .map((tag) => ( - { - e.preventDefault() - router.push(`/tag/${tag}`) - }} - > - #{tag} - - ))} - - )} -
-
- {post.metadata?.content?.summary} - {post.metadata?.content?.summary && "..."} -
-
- {post.metadata?.content.cover && ( -
- cover -
- )} - -
+ ) }), )} diff --git a/src/lib/ipfs-parser.ts b/src/lib/ipfs-parser.ts index 1340fc79..cbcc389e 100644 --- a/src/lib/ipfs-parser.ts +++ b/src/lib/ipfs-parser.ts @@ -31,4 +31,5 @@ export const toCid = (url: string) => { .replaceAll("https://cf-ipfs.com/ipfs/", "") .replaceAll("https://ipfs.4everland.xyz/ipfs/", "") .replaceAll("https://rss3.mypinata.cloud/ipfs/", "") + .replaceAll(IPFS_PREFIX, "") } diff --git a/src/models/page.model.ts b/src/models/page.model.ts index afa389a1..c65452c9 100644 --- a/src/models/page.model.ts +++ b/src/models/page.model.ts @@ -640,6 +640,10 @@ export async function getSummary({ ).data } +export async function getScore({ cid }: { cid: string }) { + return (await (await fetch(`/api/score?cid=${cid}`)).json()).data +} + export async function checkMirror(characterId: string) { const notes = await indexer.getNotes({ characterId, diff --git a/src/pages/api/score.ts b/src/pages/api/score.ts new file mode 100644 index 00000000..dec6cdc9 --- /dev/null +++ b/src/pages/api/score.ts @@ -0,0 +1,121 @@ +import { NextApiRequest, NextApiResponse } from "next" + +import { toGateway } from "~/lib/ipfs-parser" +import prisma from "~/lib/prisma.server" +import { cacheGet } from "~/lib/redis.server" + +const getOriginalScore = async (cid: string) => { + try { + const { content } = await (await fetch(toGateway(`ipfs://${cid}`))).json() + + if (content) { + console.time(`fetching score ${cid}`) + + const prompt = `According to rule 1 not too short content, rule 2 good originality and innovation, and rule 3 good fun or logic, give this article a score in the range of 0-100 and explain the reason: + "${content}" + Score:` + const response = await ( + await fetch("https://api.openai.com/v1/chat/completions", { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + }, + body: JSON.stringify({ + model: "gpt-4", + temperature: 0, + messages: [ + { + role: "user", + content: prompt, + }, + ], + }), + }) + ).json() + + console.timeEnd(`fetching score ${cid}`) + + return { + score: parseInt(response.choices?.[0]?.message?.content?.trim()), + reason: response.choices?.[0]?.message?.content + ?.trim() + .replace(/^\d+([,.\s]*)/, "") + .trim() + .replace(/^Reason:/, "") + .trim(), + } + } + } catch (error) { + console.error(error) + console.timeEnd(`fetching score ${cid}`) + } +} + +export async function getScore(cid: string) { + const score = await cacheGet({ + key: ["summary_score222", cid], + getValueFun: async () => { + const meta = await prisma.metadata.findFirst({ + where: { + uri: `ipfs://${cid}`, + }, + }) + if (meta) { + if (meta?.ai_score !== null) { + return { + score: meta.ai_score, + reason: meta.ai_score_reason, + } + } else { + const score = await getOriginalScore(cid) + if (score) { + await prisma.metadata.update({ + where: { + uri: `ipfs://${cid}`, + }, + data: { + ai_score: score.score, + ai_score_reason: score.reason, + }, + }) + + return score + } + } + } else { + const score = await getOriginalScore(cid) + if (score) { + await prisma.metadata.create({ + data: { + uri: `ipfs://${cid}`, + ai_score: score.score, + ai_score_reason: score.reason, + }, + }) + + return score + } + } + }, + noUpdate: true, + }) + + return score +} + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse, +) { + let { cid } = req.query + + if (!cid) { + res.status(400).send("Bad Request") + return + } + + res.status(200).json({ + data: await getScore(cid as string), + }) +} diff --git a/src/pages/api/summary.ts b/src/pages/api/summary.ts index dd7f807f..e07420c6 100644 --- a/src/pages/api/summary.ts +++ b/src/pages/api/summary.ts @@ -21,40 +21,44 @@ const chains = new Map() const getOriginalSummary = async (cid: string, lang: string) => { try { - console.log("fetching summary", cid, lang) - const { content } = await (await fetch(toGateway(`ipfs://${cid}`))).json() - let chain = chains.get(lang) - if (!chain) { - const prompt = new PromptTemplate({ - template: `Summarize this in ${lang} language: - "{text}" - CONCISE SUMMARY:`, - inputVariables: ["text"], + if (content) { + console.time(`fetching summary ${cid}, ${lang}`) + + let chain = chains.get(lang) + if (!chain) { + const prompt = new PromptTemplate({ + template: `Summarize this in "${lang}" language: + "{text}" + CONCISE SUMMARY:`, + inputVariables: ["text"], + }) + + const combineDocsChain = loadSummarizationChain(model, { + prompt, + combineMapPrompt: prompt, + combinePrompt: prompt, + }) + + chain = new AnalyzeDocumentChain({ + combineDocumentsChain: combineDocsChain, + }) + + chains.set(lang, chain) + } + + const res = await chain.call({ + input_document: content, }) - const combineDocsChain = loadSummarizationChain(model, { - prompt, - combineMapPrompt: prompt, - combinePrompt: prompt, - }) + console.timeEnd(`fetching summary ${cid}, ${lang}`) - chain = new AnalyzeDocumentChain({ - combineDocumentsChain: combineDocsChain, - }) - - chains.set(lang, chain) + return res?.text } - - const res = await chain.call({ - input_document: content, - }) - - return res?.text } catch (error) { console.error(error) - return undefined + console.timeEnd(`fetching summary ${cid}, ${lang}`) } } diff --git a/src/queries/page.ts b/src/queries/page.ts index 9defdb0f..52a16aa5 100644 --- a/src/queries/page.ts +++ b/src/queries/page.ts @@ -256,6 +256,17 @@ export function useGetSummary(input: { cid?: string; lang?: string }) { }) } +export function useGetScore(input: { cid?: string }) { + return useQuery(["getScore", input.cid], async () => { + if (!input.cid) { + return + } + return pageModel.getScore({ + cid: input.cid, + }) + }) +} + export function useGetMirrorXyz(input: { address: string }) { return useQuery(["getMirror", input.address], async () => { if (!input.address) {