From 8d8a435a48b0fd802085d5966b5891ced05f1d01 Mon Sep 17 00:00:00 2001 From: lucky13820 Date: Sun, 8 Mar 2026 13:13:25 -0700 Subject: [PATCH] fix(route/theinitium): rewrite for Ghost CMS after site migration (#21071) * refactor(theinitium): rewrite routes for Ghost CMS, add cookie support - Rewrite all routes to use Ghost Content API (old API deprecated) - Add INITIUM_MEMBER_COOKIE env var for paid article full-text - Add features block with requireConfig to all route files - Add url field and radar rules per RSSHub standards - Strip -zh-hans suffix from author/category display names - Update namespace with English name and zh translations - Mark follow route as deprecated (Ghost doesn't support it) - Fix hostname binding to 0.0.0.0 for LAN access * fix(theinitium): use routeExpire with refresh=false to prevent stale cache Cache was never expiring because refresh=true (default) extends TTL on every hit. Now uses 5-min routeExpire and won't extend on access. * refactor(theinitium): address review feedback - Remove unused INITIUM_USERNAME/PASSWORD/BEARER_TOKEN env keys - Remove hardcoded User-Agent (use RSSHub's internal UA) - Remove fragile html.length > 500 check - Remove refresh: false from cache.tryGet - Hoist stripLangSuffix to module scope (dedupe with cleanName) * fix(theinitium): detect paywall CTA instead of length check If cookie is invalid/expired, scrape returns paywall HTML. Check for .gh-post-upgrade-cta to detect this and fall back to cleaner Ghost API preview. * revert(index): restore '::' hostname per #16513 * feat(theinitium): clean Ghost Koenig card HTML for RSS output * fix(theinitium): use redirected Ghost API URL, remove unused config types * fix(theinitium): remove unrelated neureality config changes --------- Co-authored-by: Ryan Yao Co-authored-by: Ryan Yao Co-authored-by: Ryan Yao --- lib/config.ts | 12 +- lib/routes/theinitium/author.ts | 21 +- lib/routes/theinitium/channel.ts | 41 ++- lib/routes/theinitium/follow.ts | 45 +--- lib/routes/theinitium/namespace.ts | 18 +- lib/routes/theinitium/tags.ts | 23 +- lib/routes/theinitium/utils.ts | 415 +++++++++++++++++++---------- 7 files changed, 373 insertions(+), 202 deletions(-) diff --git a/lib/config.ts b/lib/config.ts index 3ff0d7a23..61e5204f8 100644 --- a/lib/config.ts +++ b/lib/config.ts @@ -120,9 +120,7 @@ type ConfigEnvKeys = | 'HEFENG_API_HOST' | 'HUITUN_COOKIE' | 'INFZM_COOKIE' - | 'INITIUM_USERNAME' - | 'INITIUM_PASSWORD' - | 'INITIUM_BEARER_TOKEN' + | 'INITIUM_MEMBER_COOKIE' | 'IG_USERNAME' | 'IG_PASSWORD' | 'IG_PROXY' @@ -444,9 +442,7 @@ export type Config = { cookie?: string; }; initium: { - username?: string; - password?: string; - bearertoken?: string; + memberCookie?: string; }; instagram: { username?: string; @@ -934,9 +930,7 @@ const calculateValue = () => { cookie: envs.INFZM_COOKIE, }, initium: { - username: envs.INITIUM_USERNAME, - password: envs.INITIUM_PASSWORD, - bearertoken: envs.INITIUM_BEARER_TOKEN, + memberCookie: envs.INITIUM_MEMBER_COOKIE, }, instagram: { username: envs.IG_USERNAME, diff --git a/lib/routes/theinitium/author.ts b/lib/routes/theinitium/author.ts index 253848047..d4e43d8a2 100644 --- a/lib/routes/theinitium/author.ts +++ b/lib/routes/theinitium/author.ts @@ -7,10 +7,25 @@ const handler = (ctx) => processFeed('author', ctx); export const route: Route = { path: '/author/:type/:language?', name: '作者', + url: 'theinitium.com', maintainers: ['AgFlore'], parameters: { - type: '作者 ID，可从作者主页 URL 中获取，如 `https://theinitium.com/author/ninghuilulu`', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '作者 slug，可从作者主页 URL 中获取，如 `https://theinitium.com/author/initium-newsroom/`', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { @@ -19,6 +34,6 @@ export const route: Route = { }, ], handler, - example: '/theinitium/author/ninghuilulu/zh-hans', + example: '/theinitium/author/initium-newsroom', categories: ['new-media'], }; diff --git a/lib/routes/theinitium/channel.ts b/lib/routes/theinitium/channel.ts index da63ae056..df91102c1 100644 --- a/lib/routes/theinitium/channel.ts +++ b/lib/routes/theinitium/channel.ts @@ -6,24 +6,47 @@ const handler = (ctx) => processFeed('channel', ctx); export const route: Route = { path: '/channel/:type?/:language?', - name: '专题・栏目', + name: '栏目', + url: 'theinitium.com', maintainers: ['prnake', 'mintyfrankie'], parameters: { - type: '栏目，缺省为最新', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '栏目，缺省为最新（latest）', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。获取方式：登录 theinitium.com 后，从浏览器开发者工具中复制 Cookie。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { - source: ['theinitium.com/channel/:type'], + source: ['theinitium.com/latest/'], + target: '/channel/latest', + }, + { + source: ['theinitium.com/tag/:type'], target: '/channel/:type', }, ], handler, - example: '/theinitium/channel/latest/zh-hans', + example: '/theinitium/channel/latest', categories: ['new-media'], - description: `Type 栏目： + description: `Type 栏目（对应 Ghost 标签）： -| 最新 | 深度 | What’s New | 广场 | 科技 | 风物 | 特约 | ... | -| ------ | ------- | ---------- | ----------------- | ---------- | ------- | -------- | --- | -| latest | feature | news-brief | notes-and-letters | technology | culture | pick_up | ... |`, +| 最新 | 速递 | 评论 | 国际 | 大陆 | 香港 | 台湾 | 科技 | 专题 | 日报 | 周报 | +| ------ | -------- | ------- | ------------- | -------- | -------- | ------ | ---------- | ------ | ----------- | ------ | +| latest | whatsnew | opinion | international | mainland | hongkong | taiwan | technology | feature | daily-brief | weekly | + +:::tip +设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。 +:::`, }; diff --git a/lib/routes/theinitium/follow.ts b/lib/routes/theinitium/follow.ts index 50f9ceb7c..ef56fef97 100644 --- a/lib/routes/theinitium/follow.ts +++ b/lib/routes/theinitium/follow.ts @@ -1,44 +1,19 @@ import type { Route } from '@/types'; -import { processFeed } from './utils'; - -const handler = (ctx) => processFeed('follow', ctx); - export const route: Route = { path: '/follow/articles/:language?', - name: '个人订阅追踪动态', + name: '个人订阅追踪动态（已停用）', maintainers: ['AgFlore'], parameters: { - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + language: '语言', }, - radar: [ - { - title: '作者', - source: ['theinitium.com/author/:type'], - target: '/author/:type', - }, - ], - handler, - example: '/theinitium/author/ninghuilulu/zh-hans', + radar: [], + handler: () => { + throw new Error('此路由已停用。端传媒迁移到 Ghost CMS 后不再支持个人追踪功能。请改用 /theinitium/channel/latest 或 /theinitium/tags/:tag 订阅。'); + }, + example: '/theinitium/follow/articles', categories: ['new-media'], - description: '需填入 Web 版认证 token, 也可选择直接在环境设置中填写明文的用户名和密码', - features: { - requireConfig: [ - { - name: 'INITIUM_BEARER_TOKEN', - optional: true, - description: `端传媒 Web 版认证 token。获取方式：登陆后打开端传媒站内任意页面，打开浏览器开发者工具中 “网络”(Network) 选项卡，筛选 URL 找到任一个地址为 \`api.initium.com\` 开头的请求，点击检查其 “消息头”，在 “请求头” 中找到Authorization字段，将其值复制填入配置即可。你的配置应该形如 \`INITIUM_BEARER_TOKEN: 'Bearer eyJxxxx......xx_U8'\`。使用 token 部署的好处是避免占据登陆设备数的额度，但这个 token 一般有效期为两周，因此只可作临时测试使用。`, - }, - { - name: 'INITIUM_USERNAME', - optional: true, - description: `端传媒用户名 （邮箱）`, - }, - { - name: 'INITIUM_PASSWORD', - optional: true, - description: `端传媒密码`, - }, - ], - }, + description: `:::warning +此路由已停用。端传媒已迁移到 Ghost CMS，不再支持通过 API 获取个人追踪内容。请改用标签或栏目订阅。 +:::`, }; diff --git a/lib/routes/theinitium/namespace.ts b/lib/routes/theinitium/namespace.ts index 6abfdcba1..14056e91f 100644 --- a/lib/routes/theinitium/namespace.ts +++ b/lib/routes/theinitium/namespace.ts @@ -1,12 +1,20 @@ import type { Namespace } from '@/types'; export const namespace: Namespace = { - name: '端传媒', + name: 'The Initium', url: 'theinitium.com', - description: `通过提取文章全文，以提供比官方源更佳的阅读体验。 + description: `:::tip +Set the environment variable \`INITIUM_MEMBER_COOKIE\` to get the full text of paid articles. After logging in to theinitium.com, copy the Cookie from the browser developer tools. -::: warning -付费内容全文可能需要登陆获取，详情见部署页面的配置模块。 +Old environment variables \`INITIUM_USERNAME\`, \`INITIUM_PASSWORD\`, and \`INITIUM_BEARER_TOKEN\` are no longer used since the site migrated to Ghost CMS. :::`, - lang: 'zh-HK', + + zh: { + name: '端傳媒', + description: `:::tip +设置环境变量 \`INITIUM_MEMBER_COOKIE\` 可获取付费文章全文。登录 theinitium.com 后，从浏览器开发者工具中复制 Cookie。 + +旧的环境变量 \`INITIUM_USERNAME\`、\`INITIUM_PASSWORD\` 和 \`INITIUM_BEARER_TOKEN\` 已不再使用（网站已迁移至 Ghost CMS）。 +:::`, + }, }; diff --git a/lib/routes/theinitium/tags.ts b/lib/routes/theinitium/tags.ts index fb209acda..37155f198 100644 --- a/lib/routes/theinitium/tags.ts +++ b/lib/routes/theinitium/tags.ts @@ -7,18 +7,33 @@ const handler = (ctx) => processFeed('tags', ctx); export const route: Route = { path: '/tags/:type/:language?', name: '话题・标签', + url: 'theinitium.com', maintainers: ['AgFlore'], parameters: { - type: '话题 ID，可从话题页 URL 中获取，如 `https://theinitium.com/tags/2019_10/`', - language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为简体', + type: '标签 slug，可从标签页 URL 中获取，如 `https://theinitium.com/tag/south-korea/` 则为 `south-korea`', + language: '语言，简体`zh-hans`，繁体`zh-hant`，缺省为不限', + }, + features: { + requireConfig: [ + { + name: 'INITIUM_MEMBER_COOKIE', + optional: true, + description: '端传媒会员登录后的 Cookie，用于获取付费文章全文。', + }, + ], + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, }, radar: [ { - source: ['theinitium.com/tags/:type'], + source: ['theinitium.com/tag/:type'], target: '/tags/:type', }, ], handler, - example: '/theinitium/tags/2019_10/zh-hans', + example: '/theinitium/tags/south-korea', categories: ['new-media'], }; diff --git a/lib/routes/theinitium/utils.ts b/lib/routes/theinitium/utils.ts index 902ff5aed..e630d9197 100644 --- a/lib/routes/theinitium/utils.ts +++ b/lib/routes/theinitium/utils.ts @@ -1,168 +1,309 @@ import { load } from 'cheerio'; import type { Context } from 'hono'; -import { FetchError } from 'ofetch'; import { config } from '@/config'; import InvalidParameterError from '@/errors/types/invalid-parameter'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import logger from '@/utils/logger'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; -const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY='; +// Strip '-zh-hans' suffix from display names for cleanliness +const stripLangSuffix = (name: string) => name.replace(/-zh-hans$/i, ''); -export const processFeed = async (model: string, ctx: Context) => { - // model是channel/tag/etc.，而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话，那就是model是follow，type是articles。 - const type = ctx.req.param('type') ?? 'latest'; - const language = ctx.req.param('language') ?? 'zh-hans'; - let listUrl; - let listLink; - switch (model) { - case 'author': - listUrl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/author/${type}/`; - break; - case 'follow': - listUrl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`; - listLink = `https://theinitium.com/follow/`; - break; - case 'channel': - listUrl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/channel/${type}/`; - break; - case 'tags': - listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`; - listLink = `https://theinitium.com/tags/${type}/`; - break; - default: - throw new InvalidParameterError('wrong model'); +const GHOST_API_BASE = 'https://production-initium-media.ghost.io/ghost/api/content'; +const GHOST_CONTENT_KEY = 'a44a0409c222328d39e2c75293'; + +// Old channel slugs → Ghost tag slugs mapping +const CHANNEL_TAG_MAP: Record = { + latest: '', // no filter = latest + whatsnew: 'whatsnew', + 'news-brief': 'whatsnew', + opinion: 'opinion', + international: 'international', + mainland: 'mainland', + hongkong: 'hong-kong', + taiwan: 'taiwan', + technology: 'technology', + feature: 'report', + report: 'report', + 'daily-brief': 'daily-brief', + weekly: 'weekly', +}; + +// Ghost uses a language-based tagging system: +// - zh-hant (Traditional Chinese): uses base tag slug, e.g. "whatsnew", with internal tag #zh-hant +// - zh-hans (Simplified Chinese): uses suffixed tag slug, e.g. "whatsnew-zh-hans", with internal tag #zh-hans +// When no language is specified, we return all posts (both zh-hans and zh-hant mixed). +function applyLanguageToTagSlug(tagSlug: string, language: string): string { + if (language === 'zh-hans') { + return `${tagSlug}-zh-hans`; } + // zh-hant uses the base slug + return tagSlug; +} - const key = { - email: config.initium.username, - password: config.initium.password, +interface GhostPost { + id: string; + uuid: string; + slug: string; + title: string; + html: string; + feature_image?: string; + feature_image_caption?: string; + custom_excerpt?: string; + published_at: string; + updated_at: string; + url: string; + excerpt?: string; + access: boolean; + visibility?: string; + authors?: Array<{ name: string; slug: string }>; + tags?: Array<{ name: string; slug: string; visibility: string }>; + primary_author?: { name: string; slug: string }; + primary_tag?: { name: string; slug: string }; +} + +interface GhostResponse { + posts: GhostPost[]; + meta: { + pagination: { + page: number; + limit: number; + pages: number; + total: number; + }; }; - const body = JSON.stringify(key); +} - let token; - const cacheIn = await cache.get('initium:token'); - if (cacheIn) { - token = cacheIn; - } else if (config.initium.bearertoken) { - token = config.initium.bearertoken; - cache.set('initium:token', config.initium.bearertoken); - } else if (key.email === undefined) { - token = TOKEN; - } else { - const login = await got.post(`https://api.theinitium.com/api/v2/auth/login/?language=${language}`, { - headers: { - 'Content-Type': 'application/json', - Accept: 'application/json', - Connection: 'keep-alive', - Authorization: TOKEN, - }, - body, - }); - - token = 'token ' + login.data.token; - cache.set('initium:token', token); +async function ghostFetch(endpoint: string, params: Record = {}): Promise { + const url = new URL(`${GHOST_API_BASE}/${endpoint}/`); + url.searchParams.set('key', GHOST_CONTENT_KEY); + for (const [k, v] of Object.entries(params)) { + url.searchParams.set(k, v); } + return await ofetch(url.href); +} - const headers = { - Accept: '*/*', - Connection: 'keep-alive', - Authorization: token, - }; - - let response; +async function scrapeFullArticle(url: string, cookie: string): Promise { try { - response = await got(listUrl, { - headers, + const response = await ofetch(url, { + headers: { + Cookie: cookie, + }, + parseResponse: (txt) => txt, }); - } catch (error) { - if (error instanceof FetchError && error.statusCode === 401) { - // 401 说明 token 过期了，将它删掉 - await cache.set('initium:token', ''); + const $ = load(response); + const article = $('article'); + if (article.length === 0) { + return null; } - throw error; + // If paywall CTA present, cookie didn't work — fall back to Ghost preview + if (article.find('.gh-post-upgrade-cta').length > 0) { + return null; + } + return article.html(); + } catch (error) { + logger.warn(`Failed to scrape Initium article: ${url}`, error); + return null; } +} - const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪'; - // 从v1直升的channel和tags里面是digests，v2新增的author和follow出来都是results - const articles = response.data.results ?? response.data.digests; - // 如果model=author，那就是avatar；否则都是cover，要么就没封面 - const image = response.data[model] && (response.data[model].cover || response.data[model].avatar); +/** + * Clean Ghost Koenig editor card HTML for RSS consumption. + * Strips kg-* wrapper divs, converts bookmark cards to simple links, + * removes callout markup, etc. + */ +function cleanGhostHtml(html: string): string { + const $ = load(html, null, false); - const getFullText = (slug) => - cache.tryGet(`theinitium:${slug}:${language}`, async () => { - let content = ''; - const { data } = await got(`https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`, { - headers, - }); + // Convert kg-bookmark-card to a simple link + $('a.kg-bookmark-container, a.kg-bookmark-card').each((_, el) => { + const $el = $(el); + const href = $el.attr('href') || ''; + const title = $el.find('.kg-bookmark-title').text().trim(); + const desc = $el.find('.kg-bookmark-description').text().trim(); + const replacement = title ? `

${title}${desc ? ` — ${desc}` : ''}

` : `

${href}

`; + $el.replaceWith(replacement); + }); - if (data.lead.length) { - content += '

「' + data.lead + '」

'; - } - if (data.byline.length) { - content += '

' + data.byline + '

'; - } - if (data.content) { - content += data.content.replace('