fix(route/theinitium): rewrite for Ghost CMS after site migration (#21071)
* refactor(theinitium): rewrite routes for Ghost CMS, add cookie support - Rewrite all routes to use Ghost Content API (old API deprecated) - Add INITIUM_MEMBER_COOKIE env var for paid article full-text - Add features block with requireConfig to all route files - Add url field and radar rules per RSSHub standards - Strip -zh-hans suffix from author/category display names - Update namespace with English name and zh translations - Mark follow route as deprecated (Ghost doesn't support it) - Fix hostname binding to 0.0.0.0 for LAN access * fix(theinitium): use routeExpire with refresh=false to prevent stale cache Cache was never expiring because refresh=true (default) extends TTL on every hit. Now uses 5-min routeExpire and won't extend on access. * refactor(theinitium): address review feedback - Remove unused INITIUM_USERNAME/PASSWORD/BEARER_TOKEN env keys - Remove hardcoded User-Agent (use RSSHub's internal UA) - Remove fragile html.length > 500 check - Remove refresh: false from cache.tryGet - Hoist stripLangSuffix to module scope (dedupe with cleanName) * fix(theinitium): detect paywall CTA instead of length check If cookie is invalid/expired, scrape returns paywall HTML. Check for .gh-post-upgrade-cta to detect this and fall back to cleaner Ghost API preview. * revert(index): restore '::' hostname per #16513 * feat(theinitium): clean Ghost Koenig card HTML for RSS output * fix(theinitium): use redirected Ghost API URL, remove unused config types * fix(theinitium): remove unrelated neureality config changes --------- Co-authored-by: Ryan Yao <ryanyao@Macmini.lan> Co-authored-by: Ryan Yao <ryanyao@Ryans-Mac-mini.lan> Co-authored-by: Ryan Yao <ryanyao@Ryans-Mac-mini.local>
This commit is contained in:
parent
5277218f01
commit
8d8a435a48
|
|
@ -120,9 +120,7 @@ type ConfigEnvKeys =
|
|||
| 'HEFENG_API_HOST'
|
||||
| 'HUITUN_COOKIE'
|
||||
| 'INFZM_COOKIE'
|
||||
| 'INITIUM_USERNAME'
|
||||
| 'INITIUM_PASSWORD'
|
||||
| 'INITIUM_BEARER_TOKEN'
|
||||
| 'INITIUM_MEMBER_COOKIE'
|
||||
| 'IG_USERNAME'
|
||||
| 'IG_PASSWORD'
|
||||
| 'IG_PROXY'
|
||||
|
|
@ -444,9 +442,7 @@ export type Config = {
|
|||
cookie?: string;
|
||||
};
|
||||
initium: {
|
||||
username?: string;
|
||||
password?: string;
|
||||
bearertoken?: string;
|
||||
memberCookie?: string;
|
||||
};
|
||||
instagram: {
|
||||
username?: string;
|
||||
|
|
@ -934,9 +930,7 @@ const calculateValue = () => {
|
|||
cookie: envs.INFZM_COOKIE,
|
||||
},
|
||||
initium: {
|
||||
username: envs.INITIUM_USERNAME,
|
||||
password: envs.INITIUM_PASSWORD,
|
||||
bearertoken: envs.INITIUM_BEARER_TOKEN,
|
||||
memberCookie: envs.INITIUM_MEMBER_COOKIE,
|
||||
},
|
||||
instagram: {
|
||||
username: envs.IG_USERNAME,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,25 @@ const handler = (ctx) => processFeed('author', ctx);
|
|||
export const route: Route = {
|
||||
path: '/author/:type/:language?',
|
||||
name: '作者',
|
||||
url: 'theinitium.com',
|
||||
maintainers: ['AgFlore'],
|
||||
parameters: {
|
||||
type: '作者 ID,可从作者主页 URL 中获取,如 `https://theinitium.com/author/ninghuilulu`',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体',
|
||||
type: '作者 slug,可从作者主页 URL 中获取,如 `https://theinitium.com/author/initium-newsroom/`',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为不限',
|
||||
},
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'INITIUM_MEMBER_COOKIE',
|
||||
optional: true,
|
||||
description: '端传媒会员登录后的 Cookie,用于获取付费文章全文。',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
|
|
@ -19,6 +34,6 @@ export const route: Route = {
|
|||
},
|
||||
],
|
||||
handler,
|
||||
example: '/theinitium/author/ninghuilulu/zh-hans',
|
||||
example: '/theinitium/author/initium-newsroom',
|
||||
categories: ['new-media'],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,24 +6,47 @@ const handler = (ctx) => processFeed('channel', ctx);
|
|||
|
||||
export const route: Route = {
|
||||
path: '/channel/:type?/:language?',
|
||||
name: '专题・栏目',
|
||||
name: '栏目',
|
||||
url: 'theinitium.com',
|
||||
maintainers: ['prnake', 'mintyfrankie'],
|
||||
parameters: {
|
||||
type: '栏目,缺省为最新',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体',
|
||||
type: '栏目,缺省为最新(latest)',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为不限',
|
||||
},
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'INITIUM_MEMBER_COOKIE',
|
||||
optional: true,
|
||||
description: '端传媒会员登录后的 Cookie,用于获取付费文章全文。获取方式:登录 theinitium.com 后,从浏览器开发者工具中复制 Cookie。',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['theinitium.com/channel/:type'],
|
||||
source: ['theinitium.com/latest/'],
|
||||
target: '/channel/latest',
|
||||
},
|
||||
{
|
||||
source: ['theinitium.com/tag/:type'],
|
||||
target: '/channel/:type',
|
||||
},
|
||||
],
|
||||
handler,
|
||||
example: '/theinitium/channel/latest/zh-hans',
|
||||
example: '/theinitium/channel/latest',
|
||||
categories: ['new-media'],
|
||||
description: `Type 栏目:
|
||||
description: `Type 栏目(对应 Ghost 标签):
|
||||
|
||||
| 最新 | 深度 | What’s New | 广场 | 科技 | 风物 | 特约 | ... |
|
||||
| ------ | ------- | ---------- | ----------------- | ---------- | ------- | -------- | --- |
|
||||
| latest | feature | news-brief | notes-and-letters | technology | culture | pick_up | ... |`,
|
||||
| 最新 | 速递 | 评论 | 国际 | 大陆 | 香港 | 台湾 | 科技 | 专题 | 日报 | 周报 |
|
||||
| ------ | -------- | ------- | ------------- | -------- | -------- | ------ | ---------- | ------ | ----------- | ------ |
|
||||
| latest | whatsnew | opinion | international | mainland | hongkong | taiwan | technology | feature | daily-brief | weekly |
|
||||
|
||||
:::tip
|
||||
设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。
|
||||
:::`,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,44 +1,19 @@
|
|||
import type { Route } from '@/types';
|
||||
|
||||
import { processFeed } from './utils';
|
||||
|
||||
const handler = (ctx) => processFeed('follow', ctx);
|
||||
|
||||
export const route: Route = {
|
||||
path: '/follow/articles/:language?',
|
||||
name: '个人订阅追踪动态',
|
||||
name: '个人订阅追踪动态(已停用)',
|
||||
maintainers: ['AgFlore'],
|
||||
parameters: {
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体',
|
||||
language: '语言',
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
title: '作者',
|
||||
source: ['theinitium.com/author/:type'],
|
||||
target: '/author/:type',
|
||||
},
|
||||
],
|
||||
handler,
|
||||
example: '/theinitium/author/ninghuilulu/zh-hans',
|
||||
radar: [],
|
||||
handler: () => {
|
||||
throw new Error('此路由已停用。端传媒迁移到 Ghost CMS 后不再支持个人追踪功能。请改用 /theinitium/channel/latest 或 /theinitium/tags/:tag 订阅。');
|
||||
},
|
||||
example: '/theinitium/follow/articles',
|
||||
categories: ['new-media'],
|
||||
description: '需填入 Web 版认证 token, 也可选择直接在环境设置中填写明文的用户名和密码',
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'INITIUM_BEARER_TOKEN',
|
||||
optional: true,
|
||||
description: `端传媒 Web 版认证 token。获取方式:登陆后打开端传媒站内任意页面,打开浏览器开发者工具中 “网络”(Network) 选项卡,筛选 URL 找到任一个地址为 \`api.initium.com\` 开头的请求,点击检查其 “消息头”,在 “请求头” 中找到Authorization字段,将其值复制填入配置即可。你的配置应该形如 \`INITIUM_BEARER_TOKEN: 'Bearer eyJxxxx......xx_U8'\`。使用 token 部署的好处是避免占据登陆设备数的额度,但这个 token 一般有效期为两周,因此只可作临时测试使用。`,
|
||||
},
|
||||
{
|
||||
name: 'INITIUM_USERNAME',
|
||||
optional: true,
|
||||
description: `端传媒用户名 (邮箱)`,
|
||||
},
|
||||
{
|
||||
name: 'INITIUM_PASSWORD',
|
||||
optional: true,
|
||||
description: `端传媒密码`,
|
||||
},
|
||||
],
|
||||
},
|
||||
description: `:::warning
|
||||
此路由已停用。端传媒已迁移到 Ghost CMS,不再支持通过 API 获取个人追踪内容。请改用标签或栏目订阅。
|
||||
:::`,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,12 +1,20 @@
|
|||
import type { Namespace } from '@/types';
|
||||
|
||||
export const namespace: Namespace = {
|
||||
name: '端传媒',
|
||||
name: 'The Initium',
|
||||
url: 'theinitium.com',
|
||||
description: `通过提取文章全文,以提供比官方源更佳的阅读体验。
|
||||
description: `:::tip
|
||||
Set the environment variable \`INITIUM_MEMBER_COOKIE\` to get the full text of paid articles. After logging in to theinitium.com, copy the Cookie from the browser developer tools.
|
||||
|
||||
::: warning
|
||||
付费内容全文可能需要登陆获取,详情见部署页面的配置模块。
|
||||
Old environment variables \`INITIUM_USERNAME\`, \`INITIUM_PASSWORD\`, and \`INITIUM_BEARER_TOKEN\` are no longer used since the site migrated to Ghost CMS.
|
||||
:::`,
|
||||
lang: 'zh-HK',
|
||||
|
||||
zh: {
|
||||
name: '端傳媒',
|
||||
description: `:::tip
|
||||
设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。登录 theinitium.com 后,从浏览器开发者工具中复制 Cookie。
|
||||
|
||||
旧的环境变量 \`INITIUM_USERNAME\`、\`INITIUM_PASSWORD\` 和 \`INITIUM_BEARER_TOKEN\` 已不再使用(网站已迁移至 Ghost CMS)。
|
||||
:::`,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,18 +7,33 @@ const handler = (ctx) => processFeed('tags', ctx);
|
|||
export const route: Route = {
|
||||
path: '/tags/:type/:language?',
|
||||
name: '话题・标签',
|
||||
url: 'theinitium.com',
|
||||
maintainers: ['AgFlore'],
|
||||
parameters: {
|
||||
type: '话题 ID,可从话题页 URL 中获取,如 `https://theinitium.com/tags/2019_10/`',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为简体',
|
||||
type: '标签 slug,可从标签页 URL 中获取,如 `https://theinitium.com/tag/south-korea/` 则为 `south-korea`',
|
||||
language: '语言,简体`zh-hans`,繁体`zh-hant`,缺省为不限',
|
||||
},
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'INITIUM_MEMBER_COOKIE',
|
||||
optional: true,
|
||||
description: '端传媒会员登录后的 Cookie,用于获取付费文章全文。',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['theinitium.com/tags/:type'],
|
||||
source: ['theinitium.com/tag/:type'],
|
||||
target: '/tags/:type',
|
||||
},
|
||||
],
|
||||
handler,
|
||||
example: '/theinitium/tags/2019_10/zh-hans',
|
||||
example: '/theinitium/tags/south-korea',
|
||||
categories: ['new-media'],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,168 +1,309 @@
|
|||
import { load } from 'cheerio';
|
||||
import type { Context } from 'hono';
|
||||
import { FetchError } from 'ofetch';
|
||||
|
||||
import { config } from '@/config';
|
||||
import InvalidParameterError from '@/errors/types/invalid-parameter';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import logger from '@/utils/logger';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=';
|
||||
// Strip '-zh-hans' suffix from display names for cleanliness
|
||||
const stripLangSuffix = (name: string) => name.replace(/-zh-hans$/i, '');
|
||||
|
||||
export const processFeed = async (model: string, ctx: Context) => {
|
||||
// model是channel/tag/etc.,而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话,那就是model是follow,type是articles。
|
||||
const type = ctx.req.param('type') ?? 'latest';
|
||||
const language = ctx.req.param('language') ?? 'zh-hans';
|
||||
let listUrl;
|
||||
let listLink;
|
||||
switch (model) {
|
||||
case 'author':
|
||||
listUrl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`;
|
||||
listLink = `https://theinitium.com/author/${type}/`;
|
||||
break;
|
||||
case 'follow':
|
||||
listUrl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`;
|
||||
listLink = `https://theinitium.com/follow/`;
|
||||
break;
|
||||
case 'channel':
|
||||
listUrl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`;
|
||||
listLink = `https://theinitium.com/channel/${type}/`;
|
||||
break;
|
||||
case 'tags':
|
||||
listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`;
|
||||
listLink = `https://theinitium.com/tags/${type}/`;
|
||||
break;
|
||||
default:
|
||||
throw new InvalidParameterError('wrong model');
|
||||
const GHOST_API_BASE = 'https://production-initium-media.ghost.io/ghost/api/content';
|
||||
const GHOST_CONTENT_KEY = 'a44a0409c222328d39e2c75293';
|
||||
|
||||
// Old channel slugs → Ghost tag slugs mapping
|
||||
const CHANNEL_TAG_MAP: Record<string, string> = {
|
||||
latest: '', // no filter = latest
|
||||
whatsnew: 'whatsnew',
|
||||
'news-brief': 'whatsnew',
|
||||
opinion: 'opinion',
|
||||
international: 'international',
|
||||
mainland: 'mainland',
|
||||
hongkong: 'hong-kong',
|
||||
taiwan: 'taiwan',
|
||||
technology: 'technology',
|
||||
feature: 'report',
|
||||
report: 'report',
|
||||
'daily-brief': 'daily-brief',
|
||||
weekly: 'weekly',
|
||||
};
|
||||
|
||||
// Ghost uses a language-based tagging system:
|
||||
// - zh-hant (Traditional Chinese): uses base tag slug, e.g. "whatsnew", with internal tag #zh-hant
|
||||
// - zh-hans (Simplified Chinese): uses suffixed tag slug, e.g. "whatsnew-zh-hans", with internal tag #zh-hans
|
||||
// When no language is specified, we return all posts (both zh-hans and zh-hant mixed).
|
||||
function applyLanguageToTagSlug(tagSlug: string, language: string): string {
|
||||
if (language === 'zh-hans') {
|
||||
return `${tagSlug}-zh-hans`;
|
||||
}
|
||||
// zh-hant uses the base slug
|
||||
return tagSlug;
|
||||
}
|
||||
|
||||
const key = {
|
||||
email: config.initium.username,
|
||||
password: config.initium.password,
|
||||
interface GhostPost {
|
||||
id: string;
|
||||
uuid: string;
|
||||
slug: string;
|
||||
title: string;
|
||||
html: string;
|
||||
feature_image?: string;
|
||||
feature_image_caption?: string;
|
||||
custom_excerpt?: string;
|
||||
published_at: string;
|
||||
updated_at: string;
|
||||
url: string;
|
||||
excerpt?: string;
|
||||
access: boolean;
|
||||
visibility?: string;
|
||||
authors?: Array<{ name: string; slug: string }>;
|
||||
tags?: Array<{ name: string; slug: string; visibility: string }>;
|
||||
primary_author?: { name: string; slug: string };
|
||||
primary_tag?: { name: string; slug: string };
|
||||
}
|
||||
|
||||
interface GhostResponse {
|
||||
posts: GhostPost[];
|
||||
meta: {
|
||||
pagination: {
|
||||
page: number;
|
||||
limit: number;
|
||||
pages: number;
|
||||
total: number;
|
||||
};
|
||||
};
|
||||
const body = JSON.stringify(key);
|
||||
}
|
||||
|
||||
let token;
|
||||
const cacheIn = await cache.get('initium:token');
|
||||
if (cacheIn) {
|
||||
token = cacheIn;
|
||||
} else if (config.initium.bearertoken) {
|
||||
token = config.initium.bearertoken;
|
||||
cache.set('initium:token', config.initium.bearertoken);
|
||||
} else if (key.email === undefined) {
|
||||
token = TOKEN;
|
||||
} else {
|
||||
const login = await got.post(`https://api.theinitium.com/api/v2/auth/login/?language=${language}`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
Accept: 'application/json',
|
||||
Connection: 'keep-alive',
|
||||
Authorization: TOKEN,
|
||||
},
|
||||
body,
|
||||
});
|
||||
|
||||
token = 'token ' + login.data.token;
|
||||
cache.set('initium:token', token);
|
||||
async function ghostFetch(endpoint: string, params: Record<string, string> = {}): Promise<any> {
|
||||
const url = new URL(`${GHOST_API_BASE}/${endpoint}/`);
|
||||
url.searchParams.set('key', GHOST_CONTENT_KEY);
|
||||
for (const [k, v] of Object.entries(params)) {
|
||||
url.searchParams.set(k, v);
|
||||
}
|
||||
return await ofetch(url.href);
|
||||
}
|
||||
|
||||
const headers = {
|
||||
Accept: '*/*',
|
||||
Connection: 'keep-alive',
|
||||
Authorization: token,
|
||||
};
|
||||
|
||||
let response;
|
||||
async function scrapeFullArticle(url: string, cookie: string): Promise<string | null> {
|
||||
try {
|
||||
response = await got(listUrl, {
|
||||
headers,
|
||||
const response = await ofetch(url, {
|
||||
headers: {
|
||||
Cookie: cookie,
|
||||
},
|
||||
parseResponse: (txt) => txt,
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof FetchError && error.statusCode === 401) {
|
||||
// 401 说明 token 过期了,将它删掉
|
||||
await cache.set('initium:token', '');
|
||||
const $ = load(response);
|
||||
const article = $('article');
|
||||
if (article.length === 0) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
// If paywall CTA present, cookie didn't work — fall back to Ghost preview
|
||||
if (article.find('.gh-post-upgrade-cta').length > 0) {
|
||||
return null;
|
||||
}
|
||||
return article.html();
|
||||
} catch (error) {
|
||||
logger.warn(`Failed to scrape Initium article: ${url}`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪';
|
||||
// 从v1直升的channel和tags里面是digests,v2新增的author和follow出来都是results
|
||||
const articles = response.data.results ?? response.data.digests;
|
||||
// 如果model=author,那就是avatar;否则都是cover,要么就没封面
|
||||
const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
|
||||
/**
|
||||
* Clean Ghost Koenig editor card HTML for RSS consumption.
|
||||
* Strips kg-* wrapper divs, converts bookmark cards to simple links,
|
||||
* removes callout markup, etc.
|
||||
*/
|
||||
function cleanGhostHtml(html: string): string {
|
||||
const $ = load(html, null, false);
|
||||
|
||||
const getFullText = (slug) =>
|
||||
cache.tryGet(`theinitium:${slug}:${language}`, async () => {
|
||||
let content = '';
|
||||
const { data } = await got(`https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`, {
|
||||
headers,
|
||||
});
|
||||
// Convert kg-bookmark-card to a simple link
|
||||
$('a.kg-bookmark-container, a.kg-bookmark-card').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const href = $el.attr('href') || '';
|
||||
const title = $el.find('.kg-bookmark-title').text().trim();
|
||||
const desc = $el.find('.kg-bookmark-description').text().trim();
|
||||
const replacement = title ? `<p><a href="${href}">${title}</a>${desc ? ` — ${desc}` : ''}</p>` : `<p><a href="${href}">${href}</a></p>`;
|
||||
$el.replaceWith(replacement);
|
||||
});
|
||||
|
||||
if (data.lead.length) {
|
||||
content += '<p>「' + data.lead + '」</p>';
|
||||
}
|
||||
if (data.byline.length) {
|
||||
content += '<p>' + data.byline + '</p>';
|
||||
}
|
||||
if (data.content) {
|
||||
content += data.content.replace('<figure class="advertisement"/><br/>', '').replaceAll(/(?:<br>){2}-{11}<br>.*$/g, '');
|
||||
} else if (data.type === 'html') {
|
||||
// 有时候编辑部会漏录入文章信息…………扶额。所以加这一个判断,如果确实是普通html文章,但又没有内容,说明是漏了,后面还要给guid手动加个标记,以便阅读器事后重抓。
|
||||
content += '内容为空,请稍后再来';
|
||||
} else if (data.type === 'web') {
|
||||
// 有时候文章并非普通html文章,而是带有互动内容等,表现为type为web,并且content里没有内容。我们也尽力抓点东西下来。
|
||||
// 或许可能还有其他未知情况,等碰到了再说吧。先这样留空也不碍事。
|
||||
const nonhtmlcontent = await got(data.web.url);
|
||||
const webcontent = load(nonhtmlcontent.body).html();
|
||||
content += webcontent;
|
||||
}
|
||||
if (data.paywall_enabled) {
|
||||
const google_bot_ua =
|
||||
'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
|
||||
const accept_language = language + ';q=0.9';
|
||||
const pay_part = await got(`https://theinitium.com/article/${slug}/`, {
|
||||
headers: {
|
||||
'user-agent': google_bot_ua,
|
||||
'accept-language': accept_language,
|
||||
},
|
||||
});
|
||||
const $ = load(pay_part.body);
|
||||
const pay_content = $('div.paywall').html();
|
||||
if (pay_content) {
|
||||
content += pay_content.replace('<meta itemprop="isAccessibleForFree" content="false">', '');
|
||||
}
|
||||
}
|
||||
return content;
|
||||
});
|
||||
// Convert kg-callout-card: keep text, strip wrapper
|
||||
$('.kg-callout-card').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const text = $el.find('.kg-callout-text').html() || $el.html() || '';
|
||||
$el.replaceWith(`<blockquote>${text}</blockquote>`);
|
||||
});
|
||||
|
||||
// Convert kg-toggle-card: heading + content
|
||||
$('.kg-toggle-card').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const heading = $el.find('.kg-toggle-heading-text').text().trim();
|
||||
const content = $el.find('.kg-toggle-content').html() || '';
|
||||
$el.replaceWith(`${heading ? `<p><strong>${heading}</strong></p>` : ''}${content}`);
|
||||
});
|
||||
|
||||
// Unwrap remaining kg-card divs (keep inner content)
|
||||
$('.kg-card').each((_, el) => {
|
||||
const $el = $(el);
|
||||
$el.replaceWith($el.html() || '');
|
||||
});
|
||||
|
||||
// Remove figure wrapping around bookmark cards that we already replaced
|
||||
$('figure.kg-bookmark-card').each((_, el) => {
|
||||
const $el = $(el);
|
||||
$el.replaceWith($el.html() || '');
|
||||
});
|
||||
|
||||
// Strip members-only paywall markers
|
||||
$('p:contains("<!--members-only-->")').remove();
|
||||
|
||||
return $.html();
|
||||
}
|
||||
|
||||
async function postsToItems(posts: GhostPost[]) {
|
||||
const memberCookie = config.initium?.memberCookie;
|
||||
|
||||
const items = await Promise.all(
|
||||
articles
|
||||
.filter((a) => a.article)
|
||||
.slice(0, token === TOKEN ? 25 : articles.length)
|
||||
.map(async (item) => {
|
||||
item.article.date = parseDate(item.article.date);
|
||||
item.article.updated = parseDate(item.article.updated);
|
||||
const description = await getFullText(item.article.slug);
|
||||
return {
|
||||
title: item.article.headline,
|
||||
author: item.article.authors.length > 0 ? item.article.authors.map((x) => x.name).toString() : item.article.byline,
|
||||
category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
|
||||
description,
|
||||
link: new URL(item.article.url, 'https://theinitium.com').href,
|
||||
pubDate: item.article.date,
|
||||
updated: item.article.updated,
|
||||
// 如果遇到编辑部漏录入情况,则给uuid做个手脚,以便阅读器到时重抓。
|
||||
guid: description.endsWith('内容为空,请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
|
||||
};
|
||||
})
|
||||
posts.map(async (post) => {
|
||||
const authors = post.authors?.map((a) => stripLangSuffix(a.name)) ?? [];
|
||||
const categories = post.tags?.filter((t) => t.visibility === 'public').map((t) => stripLangSuffix(t.name)) ?? [];
|
||||
|
||||
let description = post.html ? cleanGhostHtml(post.html) : post.html;
|
||||
|
||||
// For paid articles with truncated content, scrape full text if cookie available
|
||||
if (!post.access && memberCookie) {
|
||||
const fullHtml = (await cache.tryGet(`theinitium:full:${post.slug}`, () => scrapeFullArticle(post.url, memberCookie), config.cache.contentExpire)) as string | null;
|
||||
if (fullHtml) {
|
||||
description = cleanGhostHtml(fullHtml);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: post.title,
|
||||
author: authors.join(', ') || post.primary_author?.name || '',
|
||||
category: categories,
|
||||
description,
|
||||
link: post.url,
|
||||
pubDate: parseDate(post.published_at),
|
||||
updated: parseDate(post.updated_at),
|
||||
guid: post.uuid,
|
||||
banner: post.feature_image ?? undefined,
|
||||
};
|
||||
})
|
||||
);
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
export const processFeed = async (model: string, ctx: Context) => {
|
||||
const type = ctx.req.param('type') ?? 'latest';
|
||||
const language = ctx.req.param('language') ?? '';
|
||||
|
||||
let filter = '';
|
||||
let listLink = '';
|
||||
let feedName = '';
|
||||
|
||||
switch (model) {
|
||||
case 'channel': {
|
||||
const baseTag = CHANNEL_TAG_MAP[type] ?? type;
|
||||
if (baseTag === '') {
|
||||
// "latest" = no tag filter, but we can still filter by language via internal tag
|
||||
if (language === 'zh-hans' || language === 'zh-hant') {
|
||||
filter = `tag:hash-${language}`;
|
||||
}
|
||||
} else {
|
||||
const tagSlug = language ? applyLanguageToTagSlug(baseTag, language) : baseTag;
|
||||
filter = `tag:${tagSlug}`;
|
||||
}
|
||||
listLink = type === 'latest' ? 'https://theinitium.com/latest/' : `https://theinitium.com/tag/${baseTag}/`;
|
||||
feedName = type;
|
||||
break;
|
||||
}
|
||||
case 'tags': {
|
||||
const tagSlug = language ? applyLanguageToTagSlug(type, language) : type;
|
||||
filter = `tag:${tagSlug}`;
|
||||
listLink = `https://theinitium.com/tag/${type}/`;
|
||||
feedName = type;
|
||||
break;
|
||||
}
|
||||
case 'author': {
|
||||
// Author slugs also have -zh-hans suffixed versions for simplified Chinese
|
||||
const authorSlug = language === 'zh-hans' ? `${type}-zh-hans` : type;
|
||||
filter = `author:${authorSlug}`;
|
||||
listLink = `https://theinitium.com/author/${type}/`;
|
||||
feedName = type;
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new InvalidParameterError(`Unsupported model: ${model}`);
|
||||
}
|
||||
|
||||
const cacheKey = `theinitium:ghost:${model}:${type}:${language}`;
|
||||
// Use routeExpire (5 min default) and refresh=false so cache actually expires
|
||||
const data = (await cache.tryGet(
|
||||
cacheKey,
|
||||
async () => {
|
||||
const params: Record<string, string> = {
|
||||
include: 'tags,authors',
|
||||
limit: '20',
|
||||
};
|
||||
if (filter) {
|
||||
params.filter = filter;
|
||||
}
|
||||
return await ghostFetch('posts', params);
|
||||
},
|
||||
config.cache.routeExpire,
|
||||
false
|
||||
)) as GhostResponse;
|
||||
|
||||
const items = await postsToItems(data.posts);
|
||||
|
||||
// Try to get a nice display name from the first post's relevant tag/author
|
||||
let displayName = feedName;
|
||||
if (data.posts.length > 0) {
|
||||
switch (model) {
|
||||
case 'channel': {
|
||||
const baseTag = CHANNEL_TAG_MAP[type] ?? type;
|
||||
if (baseTag) {
|
||||
const langTag = language === 'zh-hans' ? `${baseTag}-zh-hans` : baseTag;
|
||||
const matchedTag = data.posts[0].tags?.find((t) => t.slug === langTag || t.slug === baseTag);
|
||||
if (matchedTag) {
|
||||
displayName = stripLangSuffix(matchedTag.name);
|
||||
}
|
||||
} else {
|
||||
displayName = '最新';
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'tags': {
|
||||
const langTag = language === 'zh-hans' ? `${type}-zh-hans` : type;
|
||||
const matchedTag = data.posts[0].tags?.find((t) => t.slug === langTag || t.slug === type);
|
||||
if (matchedTag) {
|
||||
displayName = stripLangSuffix(matchedTag.name);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
case 'author': {
|
||||
const authorSlug = language === 'zh-hans' ? `${type}-zh-hans` : type;
|
||||
const matchedAuthor = data.posts[0].authors?.find((a) => a.slug === authorSlug || a.slug === type);
|
||||
if (matchedAuthor) {
|
||||
displayName = stripLangSuffix(matchedAuthor.name);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
default:
|
||||
// Do nothing
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
title: `端传媒 - ${name}`,
|
||||
title: `端傳媒 - ${displayName}`,
|
||||
link: listLink,
|
||||
icon: 'https://theinitium.com/misc/about/logo192.png',
|
||||
icon: 'https://theinitium.com/favicon.ico',
|
||||
language: language === 'zh-hans' ? 'zh-CN' : 'zh-TW',
|
||||
item: items,
|
||||
image,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue