feat: ai filtering for latest feed

prisma/migrations/20230329235052_/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Metadata" ADD COLUMN     "ai_score_reason" TEXT;

prisma/schema.prisma

@@ -17,6 +17,7 @@ model Metadata {
   ai_summary_zhtw   String?
   ai_summary_ja   String?
   ai_score     Int?
+  ai_score_reason String?
   @@id([uri])
   @@index([uri])
 }

src/components/main/MainFeed.tsx

@@ -3,13 +3,130 @@ import { useDate } from "~/hooks/useDate"
 import { EmptyState } from "../ui/EmptyState"
 import { useRouter } from "next/router"
 import { Image } from "~/components/ui/Image"
-import { Button } from "~/components/ui/Button"
 import { useTranslation } from "next-i18next"
-import { useEffect, useState } from "react"
 import { useGetFeed } from "~/queries/home"
 import { CharacterFloatCard } from "~/components/common/CharacterFloatCard"
 import { useAccountState } from "@crossbell/connect-kit"
 import InfiniteScroll from "react-infinite-scroller"
+import { ExpandedNote } from "~/lib/types"
+import { useGetScore } from "~/queries/page"
+import { toCid } from "~/lib/ipfs-parser"
+import { cn } from "~/lib/utils"
+import { useEffect, useState } from "react"
+import { Switch } from "@headlessui/react"
+import { setStorage, getStorage } from "~/lib/storage"
+
+const Post = ({
+  post,
+  filtering,
+}: {
+  post: ExpandedNote
+  filtering: number
+}) => {
+  const router = useRouter()
+  const { t } = useTranslation(["common", "site"])
+  const date = useDate()
+  const score = useGetScore({
+    cid: toCid(post.metadata.uri || ""),
+  })
+
+  return (
+    <div
+      className={cn({
+        hidden: score.data?.score && score.data?.score <= filtering,
+      })}
+    >
+      <div className="flex items-center space-x-2">
+        <CharacterFloatCard siteId={post.character?.handle}>
+          <div className="flex items-center space-x-4 cursor-pointer">
+            <span className="w-10 h-10 inline-block">
+              <Image
+                className="rounded-full"
+                src={
+                  post.character?.metadata?.content?.avatars?.[0] ||
+                  "ipfs://bafkreiabgixxp63pg64moxnsydz7hewmpdkxxi3kdsa4oqv4pb6qvwnmxa"
+                }
+                alt={post.character?.handle || ""}
+                width="40"
+                height="40"
+              ></Image>
+            </span>
+            <span className="font-medium">
+              {post.character?.metadata?.content?.name ||
+                post.character?.handle}
+            </span>
+          </div>
+        </CharacterFloatCard>
+        <span className="text-zinc-400">·</span>
+        <time
+          dateTime={date.formatToISO(post.createdAt)}
+          className="xlog-post-date whitespace-nowrap text-zinc-400 text-sm"
+        >
+          {t("ago", {
+            time: date.dayjs
+              .duration(
+                date.dayjs(post?.createdAt).diff(date.dayjs(), "minute"),
+                "minute",
+              )
+              .humanize(),
+          })}
+        </time>
+      </div>
+      <Link
+        target="_blank"
+        href={`/api/redirection?characterId=${post.characterId}&noteId=${post.noteId}`}
+        className="xlog-post sm:hover:bg-hover bg-white transition-all p-4 ml-10 sm:rounded-xl flex flex-col sm:flex-row items-center"
+      >
+        <div className="flex-1 flex justify-center flex-col w-full min-w-0">
+          <h3 className="xlog-post-title text-2xl font-bold text-zinc-700">
+            {post.metadata?.content?.title}
+          </h3>
+          <div className="xlog-post-meta text-sm text-zinc-400 mt-1 space-x-4 flex items-center mr-8">
+            {!!post.metadata?.content?.tags?.filter(
+              (tag) => tag !== "post" && tag !== "page",
+            ).length && (
+              <span className="xlog-post-tags space-x-1 truncate min-w-0">
+                {post.metadata?.content?.tags
+                  ?.filter((tag) => tag !== "post" && tag !== "page")
+                  .map((tag) => (
+                    <span
+                      className="hover:text-zinc-600"
+                      key={tag}
+                      onClick={(e) => {
+                        e.preventDefault()
+                        router.push(`/tag/${tag}`)
+                      }}
+                    >
+                      #{tag}
+                    </span>
+                  ))}
+              </span>
+            )}
+          </div>
+          <div
+            className="xlog-post-excerpt mt-3 text-zinc-500 line-clamp-2"
+            style={{
+              wordBreak: "break-word",
+            }}
+          >
+            {post.metadata?.content?.summary}
+            {post.metadata?.content?.summary && "..."}
+          </div>
+        </div>
+        {post.metadata?.content.cover && (
+          <div className="xlog-post-cover flex items-center relative w-full sm:w-24 h-40 sm:h-24 mt-2 sm:ml-4 sm:mt-0">
+            <Image
+              className="object-cover rounded"
+              alt="cover"
+              fill={true}
+              src={post.metadata?.content.cover}
+            ></Image>
+          </div>
+        )}
+      </Link>
+    </div>
+  )
+}
 
 export const MainFeed: React.FC<{
   type?: "latest" | "recommend" | "following"
@@ -23,18 +140,14 @@ export const MainFeed: React.FC<{
     characterId: currentCharacterId,
   })
 
-  const router = useRouter()
-  const { t } = useTranslation(["common", "site"])
-  const date = useDate()
+  const hasFiltering = type === "latest"
 
-  const [isMounted, setIsMounted] = useState(false)
+  const [aiFiltering, setAiFiltering] = useState(false)
 
   useEffect(() => {
-    setIsMounted(true)
+    setAiFiltering(getStorage("ai_filtering")?.enabled || false)
   }, [])
 
-  let currentLength = 0
-
   return (
     <>
       <InfiniteScroll
@@ -49,6 +162,31 @@ export const MainFeed: React.FC<{
           </div>
         }
       >
+        {hasFiltering && (
+          <div className="mb-10 flex items-center text-zinc-500">
+            <i className="i-mingcute:android-2-line mr-2 text-lg" />
+            <span>Enable AI Filtering</span>
+            <Switch
+              checked={aiFiltering}
+              onChange={(value) => {
+                setAiFiltering(value)
+                setStorage("ai_filtering", {
+                  enabled: value,
+                })
+              }}
+              className={`${
+                aiFiltering ? "bg-accent" : "bg-gray-200"
+              } ml-5 relative inline-flex h-6 w-11 items-center rounded-full`}
+            >
+              <span className="sr-only">Enable AI Filtering</span>
+              <span
+                className={`${
+                  aiFiltering ? "translate-x-6" : "translate-x-1"
+                } inline-block h-4 w-4 transform rounded-full bg-white transition`}
+              />
+            </Switch>
+          </div>
+        )}
         {feed.isLoading ? (
           <div className="text-center">Loading...</div>
         ) : !feed.data?.pages[0]?.count ? (
@@ -57,103 +195,12 @@ export const MainFeed: React.FC<{
           <div className="xlog-posts space-y-8">
             {feed.data?.pages.map((posts) =>
               posts?.list.map((post) => {
-                currentLength++
                 return (
-                  <div key={`${post.characterId}-${post.noteId}`}>
-                    <div className="flex items-center space-x-2">
-                      <CharacterFloatCard siteId={post.character?.handle}>
-                        <div className="flex items-center space-x-4 cursor-pointer">
-                          <span className="w-10 h-10 inline-block">
-                            <Image
-                              className="rounded-full"
-                              src={
-                                post.character?.metadata?.content
-                                  ?.avatars?.[0] ||
-                                "ipfs://bafkreiabgixxp63pg64moxnsydz7hewmpdkxxi3kdsa4oqv4pb6qvwnmxa"
-                              }
-                              alt={post.character?.handle || ""}
-                              width="40"
-                              height="40"
-                            ></Image>
-                          </span>
-                          <span className="font-medium">
-                            {post.character?.metadata?.content?.name ||
-                              post.character?.handle}
-                          </span>
-                        </div>
-                      </CharacterFloatCard>
-                      <span className="text-zinc-400">·</span>
-                      <time
-                        dateTime={date.formatToISO(post.createdAt)}
-                        className="xlog-post-date whitespace-nowrap text-zinc-400 text-sm"
-                      >
-                        {t("ago", {
-                          time: date.dayjs
-                            .duration(
-                              date
-                                .dayjs(post?.createdAt)
-                                .diff(date.dayjs(), "minute"),
-                              "minute",
-                            )
-                            .humanize(),
-                        })}
-                      </time>
-                    </div>
-                    <Link
-                      target="_blank"
-                      href={`/api/redirection?characterId=${post.characterId}&noteId=${post.noteId}`}
-                      className="xlog-post sm:hover:bg-hover bg-white transition-all p-4 ml-10 sm:rounded-xl flex flex-col sm:flex-row items-center"
-                    >
-                      <div className="flex-1 flex justify-center flex-col w-full min-w-0">
-                        <h3 className="xlog-post-title text-2xl font-bold text-zinc-700">
-                          {post.metadata?.content?.title}
-                        </h3>
-                        <div className="xlog-post-meta text-sm text-zinc-400 mt-1 space-x-4 flex items-center mr-8">
-                          {!!post.metadata?.content?.tags?.filter(
-                            (tag) => tag !== "post" && tag !== "page",
-                          ).length && (
-                            <span className="xlog-post-tags space-x-1 truncate min-w-0">
-                              {post.metadata?.content?.tags
-                                ?.filter(
-                                  (tag) => tag !== "post" && tag !== "page",
-                                )
-                                .map((tag) => (
-                                  <span
-                                    className="hover:text-zinc-600"
-                                    key={tag}
-                                    onClick={(e) => {
-                                      e.preventDefault()
-                                      router.push(`/tag/${tag}`)
-                                    }}
-                                  >
-                                    #{tag}
-                                  </span>
-                                ))}
-                            </span>
-                          )}
-                        </div>
-                        <div
-                          className="xlog-post-excerpt mt-3 text-zinc-500 line-clamp-2"
-                          style={{
-                            wordBreak: "break-word",
-                          }}
-                        >
-                          {post.metadata?.content?.summary}
-                          {post.metadata?.content?.summary && "..."}
-                        </div>
-                      </div>
-                      {post.metadata?.content.cover && (
-                        <div className="xlog-post-cover flex items-center relative w-full sm:w-24 h-40 sm:h-24 mt-2 sm:ml-4 sm:mt-0">
-                          <Image
-                            className="object-cover rounded"
-                            alt="cover"
-                            fill={true}
-                            src={post.metadata?.content.cover}
-                          ></Image>
-                        </div>
-                      )}
-                    </Link>
-                  </div>
+                  <Post
+                    key={`${post.characterId}-${post.noteId}`}
+                    post={post}
+                    filtering={aiFiltering ? 60 : 0}
+                  />
                 )
               }),
             )}

src/lib/ipfs-parser.ts

@@ -31,4 +31,5 @@ export const toCid = (url: string) => {
     .replaceAll("https://cf-ipfs.com/ipfs/", "")
     .replaceAll("https://ipfs.4everland.xyz/ipfs/", "")
     .replaceAll("https://rss3.mypinata.cloud/ipfs/", "")
+    .replaceAll(IPFS_PREFIX, "")
 }

src/models/page.model.ts

@@ -640,6 +640,10 @@ export async function getSummary({
   ).data
 }
 
+export async function getScore({ cid }: { cid: string }) {
+  return (await (await fetch(`/api/score?cid=${cid}`)).json()).data
+}
+
 export async function checkMirror(characterId: string) {
   const notes = await indexer.getNotes({
     characterId,

src/pages/api/score.ts

@@ -0,0 +1,121 @@
+import { NextApiRequest, NextApiResponse } from "next"
+
+import { toGateway } from "~/lib/ipfs-parser"
+import prisma from "~/lib/prisma.server"
+import { cacheGet } from "~/lib/redis.server"
+
+const getOriginalScore = async (cid: string) => {
+  try {
+    const { content } = await (await fetch(toGateway(`ipfs://${cid}`))).json()
+
+    if (content) {
+      console.time(`fetching score ${cid}`)
+
+      const prompt = `According to rule 1 not too short content, rule 2 good originality and innovation, and rule 3 good fun or logic, give this article a score in the range of 0-100 and explain the reason:
+      "${content}"
+      Score:`
+      const response = await (
+        await fetch("https://api.openai.com/v1/chat/completions", {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${process.env.OPENAI_API_KEY}`,
+          },
+          body: JSON.stringify({
+            model: "gpt-4",
+            temperature: 0,
+            messages: [
+              {
+                role: "user",
+                content: prompt,
+              },
+            ],
+          }),
+        })
+      ).json()
+
+      console.timeEnd(`fetching score ${cid}`)
+
+      return {
+        score: parseInt(response.choices?.[0]?.message?.content?.trim()),
+        reason: response.choices?.[0]?.message?.content
+          ?.trim()
+          .replace(/^\d+([,.\s]*)/, "")
+          .trim()
+          .replace(/^Reason:/, "")
+          .trim(),
+      }
+    }
+  } catch (error) {
+    console.error(error)
+    console.timeEnd(`fetching score ${cid}`)
+  }
+}
+
+export async function getScore(cid: string) {
+  const score = await cacheGet({
+    key: ["summary_score222", cid],
+    getValueFun: async () => {
+      const meta = await prisma.metadata.findFirst({
+        where: {
+          uri: `ipfs://${cid}`,
+        },
+      })
+      if (meta) {
+        if (meta?.ai_score !== null) {
+          return {
+            score: meta.ai_score,
+            reason: meta.ai_score_reason,
+          }
+        } else {
+          const score = await getOriginalScore(cid)
+          if (score) {
+            await prisma.metadata.update({
+              where: {
+                uri: `ipfs://${cid}`,
+              },
+              data: {
+                ai_score: score.score,
+                ai_score_reason: score.reason,
+              },
+            })
+
+            return score
+          }
+        }
+      } else {
+        const score = await getOriginalScore(cid)
+        if (score) {
+          await prisma.metadata.create({
+            data: {
+              uri: `ipfs://${cid}`,
+              ai_score: score.score,
+              ai_score_reason: score.reason,
+            },
+          })
+
+          return score
+        }
+      }
+    },
+    noUpdate: true,
+  })
+
+  return score
+}
+
+export default async function handler(
+  req: NextApiRequest,
+  res: NextApiResponse,
+) {
+  let { cid } = req.query
+
+  if (!cid) {
+    res.status(400).send("Bad Request")
+    return
+  }
+
+  res.status(200).json({
+    data: await getScore(cid as string),
+  })
+}

src/pages/api/summary.ts

@@ -21,40 +21,44 @@ const chains = new Map<string, AnalyzeDocumentChain>()
 
 const getOriginalSummary = async (cid: string, lang: string) => {
   try {
-    console.log("fetching summary", cid, lang)
-
     const { content } = await (await fetch(toGateway(`ipfs://${cid}`))).json()
 
-    let chain = chains.get(lang)
-    if (!chain) {
-      const prompt = new PromptTemplate({
-        template: `Summarize this in ${lang} language:
-      "{text}"
-      CONCISE SUMMARY:`,
-        inputVariables: ["text"],
+    if (content) {
+      console.time(`fetching summary ${cid}, ${lang}`)
+
+      let chain = chains.get(lang)
+      if (!chain) {
+        const prompt = new PromptTemplate({
+          template: `Summarize this in "${lang}" language:
+        "{text}"
+        CONCISE SUMMARY:`,
+          inputVariables: ["text"],
+        })
+
+        const combineDocsChain = loadSummarizationChain(model, {
+          prompt,
+          combineMapPrompt: prompt,
+          combinePrompt: prompt,
+        })
+
+        chain = new AnalyzeDocumentChain({
+          combineDocumentsChain: combineDocsChain,
+        })
+
+        chains.set(lang, chain)
+      }
+
+      const res = await chain.call({
+        input_document: content,
       })
 
-      const combineDocsChain = loadSummarizationChain(model, {
-        prompt,
-        combineMapPrompt: prompt,
-        combinePrompt: prompt,
-      })
+      console.timeEnd(`fetching summary ${cid}, ${lang}`)
 
-      chain = new AnalyzeDocumentChain({
-        combineDocumentsChain: combineDocsChain,
-      })
-
-      chains.set(lang, chain)
+      return res?.text
     }
-
-    const res = await chain.call({
-      input_document: content,
-    })
-
-    return res?.text
   } catch (error) {
     console.error(error)
-    return undefined
+    console.timeEnd(`fetching summary ${cid}, ${lang}`)
   }
 }
 

src/queries/page.ts

@@ -256,6 +256,17 @@ export function useGetSummary(input: { cid?: string; lang?: string }) {
   })
 }
 
+export function useGetScore(input: { cid?: string }) {
+  return useQuery(["getScore", input.cid], async () => {
+    if (!input.cid) {
+      return
+    }
+    return pageModel.getScore({
+      cid: input.cid,
+    })
+  })
+}
+
 export function useGetMirrorXyz(input: { address: string }) {
   return useQuery(["getMirror", input.address], async () => {
     if (!input.address) {