xiaoxue_admin
/
Folo
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(desktop): fallback readability if server error 

- Added '@follow-app/readability' to the project and updated its import paths.
- Removed the obsolete readability utility from internal utils.
- Updated pnpm-lock.yaml to reflect new dependencies, including '@mozilla/readability' and adjustments to optional dependencies.
- Modified package.json files to include new dependencies and ensure proper workspace linking.

Signed-off-by: Innei <tukon479@gmail.com>
This commit is contained in:
Innei 2025-04-28 23:31:42 +08:00
parent 123e6937ee
commit e6aa58e25e
No known key found for this signature in database
GPG Key ID: 0F62D33977F021F7
6 changed files with 58 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
apps/desktop/layer/main/package.json
View File

@ -25,6 +25,7 @@
"@electron-toolkit/preload": "3.0.2",
"@electron-toolkit/utils": "4.0.0",
"@eneris/push-receiver": "4.3.0",
"@follow-app/readability": "workspace:*",
"@follow/shared": "workspace:*",
"@follow/utils": "workspace:*",
"@openpanel/web": "1.0.1",

2
apps/desktop/layer/main/src/tipc/reader.ts
View File

@ -2,7 +2,7 @@ import fs from "node:fs"
import path from "node:path"
import { callWindowExpose } from "@follow/shared/bridge"
import { readability } from "@follow/utils/readability"
import { readability } from "@follow-app/readability"
import { app, BrowserWindow } from "electron"
import { MsEdgeTTS, OUTPUT_FORMAT } from "msedge-tts"
import type { ModelResult } from "vscode-languagedetection"

17
apps/desktop/layer/renderer/src/hooks/biz/useEntryActions.tsx
View File

@ -18,6 +18,7 @@ import { useShowSourceContent } from "~/atoms/source-content"
import { useUserRole, whoami } from "~/atoms/user"
import { shortcuts } from "~/constants/shortcuts"
import { apiClient } from "~/lib/api-fetch"
import { tipcClient } from "~/lib/client"
import { COMMAND_ID } from "~/modules/command/commands/id"
import { getCommand, useRunCommandFn } from "~/modules/command/hooks/use-command"
import type { FollowCommandId } from "~/modules/command/types"
@ -57,9 +58,19 @@ export const toggleEntryReadability = async ({ id, url }: { id: string; url: str
})
}
} catch {
setReadabilityStatus({
[id]: ReadabilityStatus.FAILURE,
})
const result = await tipcClient?.readability({ url })
if (result) {
setReadabilityContent({
[id]: result,
})
setReadabilityStatus({
[id]: ReadabilityStatus.SUCCESS,
})
} else {
setReadabilityStatus({
[id]: ReadabilityStatus.FAILURE,
})
}
}
} else {
setReadabilityStatus({

85
packages/internal/utils/src/readability.ts
View File

@ -1,85 +0,0 @@
import { Readability } from "@mozilla/readability"
import chardet from "chardet"
import DOMPurify from "dompurify"
import { parseHTML } from "linkedom"
import { fetch } from "ofetch"
const isDev = process.env.NODE_ENV === "development"
const userAgents =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
// For avoiding xss attack from readability, the raw document string should be sanitized.
// The xss attack in electron may lead to more serious outcomes than browser environment.
// It may allows remotely execute malicious scripts in main process.
// Before the sanitizing, the DOMPurify requires a `window` environment provided by linkedom.
function sanitizeHTMLString(dirtyDocumentString: string) {
const parser = parseHTML(dirtyDocumentString)
const purify = DOMPurify(parser.window)
// How do DOMPurify changes the origin html structure,
// You can refer its document https://github.com/cure53/DOMPurify?tab=readme-ov-file#can-i-configure-dompurify
const sanitizedDocumentString = purify.sanitize(dirtyDocumentString)
return sanitizedDocumentString
}
/**
* Decodes the response body of a `fetch` request into a string, ensuring proper character set handling.
* @throws Will return "Failed to decode response content." if the decoding process encounters any errors.
*/
async function decodeResponseBodyChars(res: Response) {
// Read the response body as an ArrayBuffer
const buffer = await res.arrayBuffer()
// Step 1: Get charset from Content-Type header
const contentType = res.headers.get("content-type")
const httpCharset = contentType?.match(/charset=([\w-]+)/i)?.[1]
// Step 2: Use charset from Content-Type header or fall back to chardet
const detectedCharset = httpCharset || chardet.detect(Buffer.from(buffer)) || "utf-8"
// Step 3: Decode the response body using the detected charset
try {
const decodedText = new TextDecoder(detectedCharset, { fatal: false }).decode(buffer)
return decodedText
} catch {
return "Failed to decode response content."
}
}
export async function readability(url: string) {
const dirtyDocumentString = await fetch(url, {
headers: {
"User-Agent": userAgents,
Accept: "text/html",
},
}).then(decodeResponseBodyChars)
const sanitizedDocumentString = sanitizeHTMLString(dirtyDocumentString)
const baseUrl = new URL(url).origin
// FIXME: linkedom does not handle relative addresses in strings. Refer to
// @see https://github.com/WebReflection/linkedom/issues/153
// JSDOM handles it correctly, but JSDOM introduces canvas binding.
const { document } = parseHTML(sanitizedDocumentString)
document.querySelectorAll("a").forEach((a) => {
a.href = replaceRelativeAddress(baseUrl, a.href)
})
;(["img", "audio", "video"] as const).forEach((tag) => {
document.querySelectorAll(tag).forEach((img) => {
img.src = img.src && replaceRelativeAddress(baseUrl, img.src)
})
})
const reader = new Readability(document, {
debug: isDev,
// keep classes to set the right code language
// https://github.com/RSSNext/Follow/issues/1058
keepClasses: true,
})
return reader.parse()
}
const replaceRelativeAddress = (baseUrl: string, url: string) => {
if (url.startsWith("http")) {
return url
}
return new URL(url, baseUrl).href
}

2
packages/readability/package.json
View File

@ -12,11 +12,11 @@
},
"devDependencies": {
"@follow/configs": "workspace:*",
"@mozilla/readability": "0.6.0",
"chardet": "2.1.0",
"dompurify": "3.2.5",
"linkedom": "0.18.10",
"nbump": "2.1.2",
"readability": "0.1.0",
"tsup": "8.4.0"
},
"publishConfig": {

81
pnpm-lock.yaml
View File

@ -332,6 +332,9 @@ importers:
'@eneris/push-receiver':
specifier: 4.3.0
version: 4.3.0
'@follow-app/readability':
specifier: workspace:*
version: link:../../../../packages/readability
'@follow/shared':
specifier: workspace:*
version: link:../../../../packages/internal/shared
@ -1684,6 +1687,9 @@ importers:
'@follow/configs':
specifier: workspace:*
version: link:../configs
'@mozilla/readability':
specifier: 0.6.0
version: 0.6.0(patch_hash=61709486ef07e9c8175a4b3f19f1aa4e2a1a75a3c557643c574d6cdacd6dd959)
chardet:
specifier: 2.1.0
version: 2.1.0
@ -1696,9 +1702,6 @@ importers:
nbump:
specifier: 2.1.2
version: 2.1.2(conventional-commits-filter@5.0.0)
readability:
specifier: 0.1.0
version: 0.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5)
tsup:
specifier: 8.4.0
version: 8.4.0(jiti@2.4.2)(postcss@8.5.3)(tsx@4.19.3)(typescript@5.8.3)(yaml@2.7.1)
@ -10254,10 +10257,6 @@ packages:
htmlparser2@9.1.0:
resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==}
htmlparser@1.7.7:
resolution: {integrity: sha512-zpK66ifkT0fauyFh2Mulrq4AqGTucxGtOhZ8OjkbSfcCpkqQEI8qRkY0tSQSJNAQ4HUZkgWaU4fK4EH6SVH9PQ==}
engines: {node: '>=0.1.33'}
http-cache-semantics@4.1.1:
resolution: {integrity: sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==}
@ -11814,10 +11813,6 @@ packages:
resolution: {integrity: sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA==}
engines: {node: '>= 18'}
mjsunit.runner@0.1.3:
resolution: {integrity: sha512-m0LYKN/kv+Yx1TwYGneCsztgPp5zTa5Erx0KuF20opXjeEO+5xOCjwoaq+qx69/tte3DaFD6atgcf/JFW0l8Pw==}
engines: {node: '>=0.1.9'}
mkdirp@0.5.6:
resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==}
hasBin: true
@ -13495,10 +13490,6 @@ packages:
resolution: {integrity: sha512-9viLL4/n1BJUCT1NXVTdS1jtm80yDEgR5T4yCelII49Mbj0v1rZdKqj7zCiYdbB0CuCgdrvHcNogAKTFPBocFA==}
engines: {node: '>=18'}
readability@0.1.0:
resolution: {integrity: sha512-rt7d3Us0ev/VNBfsSdNHjWn1xibcfkvpQ/Ahtt87s9nzcIri5L4b6r3oxqBjKh1BhTwJcGk6kUd0Y6Q5jl+20Q==}
engines: {node: '>=0.3.1'}
readable-stream@2.3.8:
resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==}
@ -15810,6 +15801,7 @@ snapshots:
'@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3)
'@csstools/css-tokenizer': 3.0.3
lru-cache: 10.4.3
optional: true
'@babel/code-frame@7.10.4':
dependencies:
@ -16772,12 +16764,14 @@ snapshots:
dependencies:
'@jridgewell/trace-mapping': 0.3.9
'@csstools/color-helpers@5.0.2': {}
'@csstools/color-helpers@5.0.2':
optional: true
'@csstools/css-calc@2.1.3(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3)':
dependencies:
'@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3)
'@csstools/css-tokenizer': 3.0.3
optional: true
'@csstools/css-color-parser@3.0.9(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3)':
dependencies:
@ -16785,12 +16779,15 @@ snapshots:
'@csstools/css-calc': 2.1.3(@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3))(@csstools/css-tokenizer@3.0.3)
'@csstools/css-parser-algorithms': 3.0.4(@csstools/css-tokenizer@3.0.3)
'@csstools/css-tokenizer': 3.0.3
optional: true
'@csstools/css-parser-algorithms@3.0.4(@csstools/css-tokenizer@3.0.3)':
dependencies:
'@csstools/css-tokenizer': 3.0.3
optional: true
'@csstools/css-tokenizer@3.0.3': {}
'@csstools/css-tokenizer@3.0.3':
optional: true
'@daybrush/utils@1.13.0': {}
@ -23796,6 +23793,7 @@ snapshots:
dependencies:
'@asamuzakjp/css-color': 3.1.5
rrweb-cssom: 0.8.0
optional: true
csstype@3.1.3: {}
@ -23814,6 +23812,7 @@ snapshots:
dependencies:
whatwg-mimetype: 4.0.0
whatwg-url: 14.2.0
optional: true
data-view-buffer@1.0.2:
dependencies:
@ -23857,7 +23856,8 @@ snapshots:
decamelize@1.2.0: {}
decimal.js@10.5.0: {}
decimal.js@10.5.0:
optional: true
decode-bmp@0.2.1:
dependencies:
@ -26587,6 +26587,7 @@ snapshots:
html-encoding-sniffer@4.0.0:
dependencies:
whatwg-encoding: 3.1.1
optional: true
html-escaper@3.0.3: {}
@ -26620,8 +26621,6 @@ snapshots:
domutils: 3.2.2
entities: 4.5.0
htmlparser@1.7.7: {}
http-cache-semantics@4.1.1: {}
http-call@5.3.0:
@ -26659,6 +26658,7 @@ snapshots:
debug: 4.4.0(supports-color@8.1.1)
transitivePeerDependencies:
- supports-color
optional: true
http2-wrapper@1.0.3:
dependencies:
@ -27010,7 +27010,8 @@ snapshots:
is-plain-object@5.0.0: {}
is-potential-custom-element-name@1.0.1: {}
is-potential-custom-element-name@1.0.1:
optional: true
is-property@1.0.2:
optional: true
@ -27319,6 +27320,7 @@ snapshots:
- bufferutil
- supports-color
- utf-8-validate
optional: true
jsep@1.4.0: {}
@ -28566,8 +28568,6 @@ snapshots:
dependencies:
minipass: 7.1.2
mjsunit.runner@0.1.3: {}
mkdirp@0.5.6:
dependencies:
minimist: 1.2.8
@ -28824,7 +28824,8 @@ snapshots:
nullthrows@1.1.1: {}
nwsapi@2.2.20: {}
nwsapi@2.2.20:
optional: true
nypm@0.6.0:
dependencies:
@ -30207,17 +30208,6 @@ snapshots:
type-fest: 4.40.0
unicorn-magic: 0.1.0
readability@0.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5):
dependencies:
htmlparser: 1.7.7
jsdom: 26.1.0(bufferutil@4.0.9)(utf-8-validate@6.0.5)
mjsunit.runner: 0.1.3
transitivePeerDependencies:
- bufferutil
- canvas
- supports-color
- utf-8-validate
readable-stream@2.3.8:
dependencies:
core-util-is: 1.0.3
@ -30641,7 +30631,8 @@ snapshots:
rou3@0.5.1: {}
rrweb-cssom@0.8.0: {}
rrweb-cssom@0.8.0:
optional: true
rtl-detect@1.1.2: {}
@ -30714,6 +30705,7 @@ snapshots:
saxes@6.0.0:
dependencies:
xmlchars: 2.2.0
optional: true
scheduler@0.23.2:
dependencies:
@ -31370,7 +31362,8 @@ snapshots:
csso: 5.0.5
picocolors: 1.1.1
symbol-tree@3.2.4: {}
symbol-tree@3.2.4:
optional: true
synckit@0.10.3:
dependencies:
@ -31587,13 +31580,15 @@ snapshots:
tinyspy@3.0.2: {}
tldts-core@6.1.86: {}
tldts-core@6.1.86:
optional: true
tldts-core@7.0.4: {}
tldts@6.1.86:
dependencies:
tldts-core: 6.1.86
optional: true
tldts@7.0.4:
dependencies:
@ -31637,6 +31632,7 @@ snapshots:
tough-cookie@5.1.2:
dependencies:
tldts: 6.1.86
optional: true
tr46@0.0.3: {}
@ -31647,6 +31643,7 @@ snapshots:
tr46@5.1.1:
dependencies:
punycode: 2.3.1
optional: true
tree-kill@1.2.2: {}
@ -32355,6 +32352,7 @@ snapshots:
w3c-xmlserializer@5.0.0:
dependencies:
xml-name-validator: 5.0.0
optional: true
walker@1.0.8:
dependencies:
@ -32414,6 +32412,7 @@ snapshots:
dependencies:
tr46: 5.1.1
webidl-conversions: 7.0.0
optional: true
whatwg-url@5.0.0:
dependencies:
@ -32690,7 +32689,8 @@ snapshots:
parse-headers: 2.0.6
xtend: 4.0.2
xml-name-validator@5.0.0: {}
xml-name-validator@5.0.0:
optional: true
xml-parse-from-string@1.0.1: {}
@ -32710,7 +32710,8 @@ snapshots:
xmlbuilder@15.1.1: {}
xmlchars@2.2.0: {}
xmlchars@2.2.0:
optional: true
xss@1.0.15:
dependencies: