feat: upgrade unified, rewrite unified pipeline, rehype ipfs

package.json

@@ -57,7 +57,7 @@
     "@lezer/highlight": "1.2.0",
     "@mantine/core": "7.3.2",
     "@mantine/dates": "7.3.2",
-    "@microflash/remark-callout-directives": "1.6.0",
+    "@microflash/remark-callout-directives": "2.0.0",
     "@monaco-editor/react": "4.6.0",
     "@popperjs/core": "2.11.8",
     "@prisma/client": "5.8.1",
@@ -84,6 +84,7 @@
     "gpt-tokenizer": "^2.1.2",
     "hast-util-from-html": "2.0.1",
     "hast-util-to-html": "9.0.0",
+    "hast-util-to-jsx-runtime": "2.3.0",
     "image-size": "^1.1.1",
     "immer": "10.0.3",
     "ioredis": "5.3.2",
@@ -124,27 +125,25 @@
     "react-tweet": "3.2.0",
     "react-virtuoso": "4.6.2",
     "refractor": "4.8.1",
-    "rehype-autolink-headings": "6.1.1",
-    "rehype-infer-description-meta": "1.1.0",
-    "rehype-katex": "6.0.3",
-    "rehype-prism-plus": "1.6.3",
-    "rehype-raw": "6.1.1",
-    "rehype-react": "7.2.0",
-    "rehype-rewrite": "3.0.6",
-    "rehype-sanitize": "5.0.1",
-    "rehype-slug": "5.1.0",
-    "rehype-stringify": "9.0.4",
-    "remark-breaks": "3.0.3",
-    "remark-directive": "2.0.1",
+    "rehype-autolink-headings": "7.1.0",
+    "rehype-infer-description-meta": "2.0.0",
+    "rehype-katex": "7.0.0",
+    "rehype-prism-plus": "2.0.0",
+    "rehype-raw": "7.0.0",
+    "rehype-rewrite": "4.0.2",
+    "rehype-sanitize": "6.0.0",
+    "rehype-slug": "6.0.0",
+    "remark-breaks": "4.0.0",
+    "remark-directive": "3.0.0",
     "remark-directive-rehype": "0.4.2",
-    "remark-emoji": "4.0.0",
-    "remark-frontmatter": "4.0.1",
-    "remark-gfm": "3.0.1",
-    "remark-github-alerts": "^0.0.2",
-    "remark-math": "5.1.1",
-    "remark-parse": "10.0.2",
-    "remark-rehype": "10.1.0",
-    "remark-stringify": "10.0.3",
+    "remark-emoji": "4.0.1",
+    "remark-frontmatter": "5.0.0",
+    "remark-gfm": "4.0.0",
+    "remark-github-alerts": "^0.0.4",
+    "remark-math": "6.0.0",
+    "remark-parse": "11.0.0",
+    "remark-rehype": "11.1.0",
+    "remark-stringify": "11.0.0",
     "remove-markdown": "0.5.0",
     "serialize-javascript": "6.0.2",
     "sharp": "0.32.6",
@@ -153,10 +152,11 @@
     "tailwind-merge": "2.2.0",
     "tailwind-scrollbar-hide": "1.1.7",
     "tailwindcss-animate": "1.0.7",
-    "unified": "10.1.2",
+    "unified": "11.0.4",
     "unist-builder": "4.0.0",
     "unist-util-visit": "5.0.0",
     "uuid": "9.0.1",
+    "vfile": "6.0.1",
     "viem": "1.21.1",
     "wagmi": "1.4.13",
     "xregexp": "^5.1.1",

src/components/ui/Mention.tsx

@@ -9,12 +9,8 @@ import { CharacterFloatCard } from "../common/CharacterFloatCard"
 import { UniLink } from "./UniLink"
 
 const getSiteId = ({ id, children }: { id: string; children?: any }) => {
-  if (
-    children?.[0] &&
-    typeof children[0] === "string" &&
-    children[0].startsWith("@")
-  ) {
-    return children[0].replace(/^@/, "")
+  if (children && typeof children === "string" && children.startsWith("@")) {
+    return children.replace(/^@/, "")
   } else if (id) {
     return id.replace(/^user-content-/, "")
   }

src/markdown/index.ts

@@ -1,23 +1,25 @@
+import type { Root as HashRoot } from "hast"
+import { toHtml } from "hast-util-to-html"
+import { toJsxRuntime } from "hast-util-to-jsx-runtime"
 import jsYaml from "js-yaml"
-import type { Root } from "mdast"
+import type { Root as MdashRoot } from "mdast"
 import { toc, Result as TocResult } from "mdast-util-toc"
 import dynamic from "next/dynamic"
-import { createElement, ReactElement } from "react"
 import { toast } from "react-hot-toast"
+// @ts-expect-error: untyped.
+import { Fragment, jsx, jsxs } from "react/jsx-runtime"
 import { refractor } from "refractor"
-import jsx from "refractor/lang/jsx"
-import solidity from "refractor/lang/solidity"
-import tsx from "refractor/lang/tsx"
+import langJsx from "refractor/lang/jsx"
+import langSolidity from "refractor/lang/solidity"
+import langTsx from "refractor/lang/tsx"
 import rehypeAutolinkHeadings from "rehype-autolink-headings"
 import rehypeInferDescriptionMeta from "rehype-infer-description-meta"
 import rehypeKatex from "rehype-katex"
 import rehypePrismGenerator from "rehype-prism-plus/generator"
-import rehypeRaw from "rehype-raw"
-import rehypeReact from "rehype-react"
+import rehypeRaw, { Options as RehypeRawOptions } from "rehype-raw"
 import rehypeRewrite from "rehype-rewrite"
 import rehypeSanitize from "rehype-sanitize"
 import rehypeSlug from "rehype-slug"
-import rehypeStringify from "rehype-stringify"
 import remarkBreaks from "remark-breaks"
 import remarkDirective from "remark-directive"
 import remarkDirectiveRehype from "remark-directive-rehype"
@@ -29,23 +31,24 @@ import remarkMath from "remark-math"
 import remarkParse from "remark-parse"
 import remarkRehype from "remark-rehype"
 import { unified } from "unified"
+import { visit } from "unist-util-visit"
+import { VFile } from "vfile"
 
 // @ts-expect-error
 import remarkCalloutDirectives from "@microflash/remark-callout-directives"
 
 import AdvancedImage from "~/components/ui/AdvancedImage"
+import { isServerSide } from "~/lib/utils"
 
 import { transformers } from "./embed-transformers"
-import { rehypeAudio } from "./rehype-audio"
 import {
   allowedCustomWrappers,
   defaultRules,
   rehypeCustomWrapper,
 } from "./rehype-custom-wrapper"
 import { rehypeEmbed } from "./rehype-embed"
-import { rehypeImage } from "./rehype-image"
+import { rehypeIpfs } from "./rehype-ipfs"
 import { rehypeTable } from "./rehype-table"
-import { rehypeVideo } from "./rehype-video"
 import { rehypeWrapCode } from "./rehype-wrap-code"
 import { rehypeExternalLink } from "./rehyper-external-link"
 import { remarkMermaid } from "./remark-mermaid"
@@ -60,70 +63,27 @@ const Tweet = dynamic(() => import("~/components/ui/Tweet"))
 const GithubRepo = dynamic(() => import("~/components/ui/GithubRepo"))
 const XLogPost = dynamic(() => import("~/components/ui/XLogPost"))
 const APlayer = dynamic(() => import("~/components/ui/APlayer"))
-
 const DPlayer = dynamic(() => import("~/components/ui/DPlayer"))
 
-export type MarkdownEnv = {
-  excerpt: string
-  frontMatter: Record<string, any>
-  __internal: Record<string, any>
-  cover: string
-  images: string[]
-  audio: string
-  toc: TocResult | null
-  tree: Root | null
-}
-
-export type Rendered = {
-  contentHTML: string
-  element?: ReactElement
-} & Omit<MarkdownEnv, "__internal">
-
 refractor.alias("html", ["svelte", "vue"])
-refractor.register(tsx)
-refractor.register(jsx)
-refractor.register(solidity)
-
+refractor.register(langTsx)
+refractor.register(langJsx)
+refractor.register(langSolidity)
 const rehypePrism = rehypePrismGenerator(refractor)
 
 export const renderPageContent = (
   content: string,
-  html?: boolean,
-  simple?: boolean,
-): Rendered => {
-  const env: MarkdownEnv = {
-    excerpt: "",
-    __internal: {},
-    frontMatter: {},
-    cover: "",
-    images: [],
-    audio: "",
-    toc: null,
-    tree: null,
-  }
-
-  let contentHTML = ""
-  let result: any = null
+  htmlMode?: boolean,
+  strictMode?: boolean,
+) => {
+  let hastTree: HashRoot | undefined = undefined
+  let mdastTree: MdashRoot | undefined = undefined
   try {
-    let pipeline = unified()
+    const pipeline = unified()
       .use(remarkParse)
-      // make sure this is before remarkBreaks
-      .use(remarkGithubAlerts)
+      .use(remarkGithubAlerts) // make sure this is before remarkBreaks
       .use(remarkBreaks)
       .use(remarkFrontmatter, ["yaml"])
-      .use(() => (tree: Root) => {
-        const yaml = tree.children.find((node) => node.type === "yaml")
-        if ((yaml as any)?.value) {
-          try {
-            env.frontMatter = jsYaml.load((yaml as any)?.value) as Record<
-              string,
-              any
-            >
-          } catch (e) {
-            console.error(e)
-          }
-        }
-      })
       .use(remarkGfm, {
         singleTilde: false,
       })
@@ -136,29 +96,17 @@ export const renderPageContent = (
         singleDollarTextMath: false,
       })
       .use(remarkPangu)
-      .use(() => (tree) => {
-        env.toc = toc(tree as any, { tight: true, ordered: true })
-      })
-      .use(remarkRehype, { allowDangerousHtml: true })
       .use(emoji)
-
-    if (!html) {
-      pipeline.use(rehypeCustomWrapper, {
+      .use(remarkRehype, { allowDangerousHtml: true })
+      .use(rehypeCustomWrapper, {
         rules: defaultRules,
       })
-    }
-
-    pipeline
-      .use(rehypeStringify)
-      .use(rehypeRaw, { passThrough: allowedCustomWrappers })
-      .use(rehypeImage, { env })
-      .use(rehypeAudio, { env })
-      .use(rehypeVideo, { env })
+      .use(rehypeIpfs)
       .use(rehypeSlug)
       .use(rehypeAutolinkHeadings, {
         behavior: "append",
         properties: {
-          className: ["xlog-anchor"],
+          className: "xlog-anchor",
           ariaHidden: true,
           tabIndex: -1,
         },
@@ -171,7 +119,7 @@ export const renderPageContent = (
           ]
         },
       })
-      .use(rehypeSanitize, simple ? undefined : sanitizeScheme)
+      .use(rehypeSanitize, strictMode ? undefined : sanitizeScheme)
       .use(rehypeTable)
       .use(rehypeExternalLink)
       .use(rehypeWrapCode)
@@ -216,53 +164,99 @@ export const renderPageContent = (
           }
         },
       })
-      .use(rehypePrism, {
-        ignoreMissing: true,
-        showLineNumbers: true,
-      })
+      .use(rehypeRaw, {
+        passThrough: allowedCustomWrappers,
+      } as RehypeRawOptions)
       // Move it to the end as it generates a lot of DOM and requires extensive traversal.
       .use(rehypeKatex, {
         strict: false,
       })
-
-    if (!html) {
-      pipeline.use(rehypeReact, {
-        createElement: createElement,
-        components: {
-          img: AdvancedImage,
-          mention: Mention,
-          mermaid: Mermaid,
-          audio: APlayer,
-          video: DPlayer,
-          tweet: Tweet,
-          "github-repo": GithubRepo,
-          "xlog-post": XLogPost,
-          style: Style,
-        } as any,
+      .use(rehypePrism, {
+        ignoreMissing: true,
+        showLineNumbers: true,
       })
-    }
 
-    result = pipeline
-      .use(() => (tree: Root) => {
-        env.tree = tree
-      })
-      .processSync(content)
+    const file = new VFile(content)
 
-    contentHTML = result.toString()
+    // markdown abstract syntax tree
+    mdastTree = pipeline.parse(file)
+    // hypertext abstract syntax tree
+    hastTree = pipeline.runSync(mdastTree, file)
   } catch (e) {
     const error = e as Error
-    toast.error(error?.message)
     console.error(e)
+    if (!isServerSide()) {
+      toast.error(error?.message)
+    }
   }
+
+  let html: string | undefined = undefined
+  let element: JSX.Element | undefined = undefined
+  let frontMatter: Record<string, any> | undefined = undefined
+  let tocResult: TocResult | undefined = undefined
+  let images: string[] = []
+  let audio: string | undefined = undefined
+
+  if (hastTree && mdastTree) {
+    // frontMatter
+    visit(mdastTree, (node, index, parent) => {
+      if (node.type === "yaml") {
+        frontMatter = jsYaml.load(node.value) as Record<string, any>
+      }
+    })
+
+    visit(hastTree, (node, index, parent) => {
+      if (node.type === "element") {
+        if (node.tagName === "img" && typeof node.properties.src === "string") {
+          images.push(node.properties.src)
+        }
+        if (node.tagName === "audio") {
+          if (typeof node.properties.cover === "string") {
+            images.push(node.properties.cover)
+          }
+          if (!audio && typeof node.properties.src === "string") {
+            audio = node.properties.src
+          }
+        }
+      }
+    })
+
+    tocResult = toc(mdastTree, {
+      tight: true,
+      ordered: true,
+    })
+
+    html = toHtml(hastTree)
+
+    element = toJsxRuntime(hastTree, {
+      Fragment,
+      components: {
+        img: AdvancedImage,
+        mention: Mention,
+        mermaid: Mermaid,
+        audio: APlayer,
+        video: DPlayer,
+        tweet: Tweet,
+        "github-repo": GithubRepo,
+        "xlog-post": XLogPost,
+        style: Style,
+      },
+      ignoreInvalidStyle: true,
+      jsx,
+      jsxs,
+      passKeys: true,
+      passNode: true,
+    })
+  }
+
   return {
-    contentHTML,
-    element: result?.result,
-    excerpt: result?.data.meta.description,
-    frontMatter: env.frontMatter,
-    cover: env.cover,
-    images: env.images,
-    audio: env.audio,
-    toc: env.toc,
-    tree: env.tree,
+    html,
+    element,
+    frontMatter,
+    toc: tocResult,
+    cover: images[0],
+    images,
+    audio,
+    tree: {},
   }
 }

src/markdown/rehype-audio.ts

@@ -1,63 +0,0 @@
-import type { Root } from "hast"
-import { Plugin } from "unified"
-import { visit } from "unist-util-visit"
-
-import { toGateway } from "~/lib/ipfs-parser"
-
-import { MarkdownEnv } from "."
-
-export const rehypeAudio: Plugin<Array<{ env: MarkdownEnv }>, Root> = ({
-  env,
-}) => {
-  return (tree: Root) => {
-    let first = true
-    visit(tree, { tagName: "audio" }, (node, i, parent) => {
-      if (!node.properties) {
-        return
-      }
-
-      const src = node.properties.src
-
-      if (!src || typeof src !== "string") {
-        return
-      }
-
-      node.properties.src = toGateway(src)
-
-      if (node.properties.cover && typeof node.properties.cover === "string") {
-        const coverIpfsUrl = toGateway(node.properties.cover)
-        node.properties.cover = coverIpfsUrl
-        env.images.push(coverIpfsUrl)
-      }
-
-      if (first) {
-        if (
-          !env.cover &&
-          node.properties.cover &&
-          typeof node.properties.cover === "string"
-        ) {
-          env.cover = node.properties.cover
-        }
-        env.audio = node.properties.src
-        first = false
-      }
-
-      if (
-        parent &&
-        parent.type === "element" &&
-        i !== null &&
-        i !== undefined
-      ) {
-        parent.tagName = "div"
-        parent.children[i] = {
-          type: "element",
-          tagName: "audio",
-          properties: {
-            ...node.properties,
-          },
-          children: [],
-        }
-      }
-    })
-  }
-}

src/markdown/rehype-image.ts

@@ -1,47 +0,0 @@
-import type { Root } from "hast"
-import { Plugin } from "unified"
-import { visit } from "unist-util-visit"
-
-import { IS_PROD } from "~/lib/constants"
-import { toGateway } from "~/lib/ipfs-parser"
-
-import { MarkdownEnv } from "."
-
-const isExternLink = (url: string) => /^https?:\/\//.test(url)
-
-export const rehypeImage: Plugin<Array<{ env: MarkdownEnv }>, Root> = ({
-  env,
-}) => {
-  return (tree: Root) => {
-    let first = true
-    visit(tree, { tagName: "img" }, (node) => {
-      if (!node.properties) return
-
-      let url = node.properties.src
-
-      if (!url || typeof url !== "string") {
-        return
-      }
-
-      const ipfsUrl = toGateway(url)
-      node.properties.src = ipfsUrl
-      url = ipfsUrl
-
-      if (first) {
-        env.cover = url
-        first = false
-      }
-      env.images.push(url)
-
-      if (isExternLink(url)) {
-        if (!url.startsWith("https:") && IS_PROD) {
-          console.error(`External image url must start with https: ${url}`)
-          url.replace(/^http:/, "https:")
-        }
-        return
-      }
-
-      node.properties.src = url
-    })
-  }
-}

src/markdown/rehype-ipfs.ts

@@ -0,0 +1,21 @@
+import type { Root } from "hast"
+import { Plugin } from "unified"
+import { visit } from "unist-util-visit"
+
+import { toGateway } from "~/lib/ipfs-parser"
+
+const tags = ["video", "audio", "img", "source"]
+
+export const rehypeIpfs: Plugin<Array<void>, Root> = () => {
+  return (tree: Root) => {
+    visit(tree, (node, i, parent) => {
+      if (node.type === "element") {
+        if (tags.includes(node.tagName)) {
+          if (typeof node.properties.src === "string") {
+            node.properties.src = toGateway(node.properties.src)
+          }
+        }
+      }
+    })
+  }
+}

src/markdown/rehype-video.ts

@@ -1,36 +0,0 @@
-import type { Root } from "hast"
-import { Plugin } from "unified"
-import { visit } from "unist-util-visit"
-
-import { toGateway } from "~/lib/ipfs-parser"
-
-import { MarkdownEnv } from "."
-
-export const rehypeVideo: Plugin<Array<{ env: MarkdownEnv }>, Root> = ({
-  env,
-}) => {
-  return (tree: Root) => {
-    let first = true
-    visit(tree, { tagName: "video" }, (node, i, parent) => {
-      if (!node.properties) {
-        return
-      }
-
-      for (const child of node.children) {
-        if (child.type === "element" && child.tagName === "source") {
-          if (!child.properties) {
-            continue
-          }
-
-          const src = child.properties.src
-
-          if (!src || typeof src !== "string") {
-            continue
-          }
-
-          child.properties.src = toGateway(src)
-        }
-      }
-    })
-  }
-}