From 4655e0952b503c178f7f3271a0b38ae65c590cc2 Mon Sep 17 00:00:00 2001
From: Ethan Shen <42264778+nczitzk@users.noreply.github.com>
Date: Thu, 11 Apr 2024 03:35:55 +0800
Subject: [PATCH] feat(route): add WordPress (#15171)
* feat(route): add WordPress
* fix docs
* feat: add search keywords
---
lib/routes-deprecated/blogs/wordpress.js | 50 ----
lib/routes/wordpress/index.ts | 172 +++++++++++++
lib/routes/wordpress/namespace.ts | 8 +
lib/routes/wordpress/util.ts | 307 +++++++++++++++++++++++
4 files changed, 487 insertions(+), 50 deletions(-)
delete mode 100644 lib/routes-deprecated/blogs/wordpress.js
create mode 100644 lib/routes/wordpress/index.ts
create mode 100644 lib/routes/wordpress/namespace.ts
create mode 100644 lib/routes/wordpress/util.ts
diff --git a/lib/routes-deprecated/blogs/wordpress.js b/lib/routes-deprecated/blogs/wordpress.js
deleted file mode 100644
index d0d0a8c97..000000000
--- a/lib/routes-deprecated/blogs/wordpress.js
+++ /dev/null
@@ -1,50 +0,0 @@
-const parser = require('@/utils/rss-parser');
-const config = require('@/config').value;
-const allowDomain = new Set(['lawrence.code.blog']);
-
-module.exports = async (ctx) => {
- if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(ctx.params.domain)) {
- ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
- }
-
- const scheme = ctx.params.https || 'https';
- const cdn = config.wordpress.cdnUrl;
-
- const domain = `${scheme}://${ctx.params.domain}`;
- const feed = await parser.parseURL(`${domain}/feed/`);
- const items = await Promise.all(
- feed.items.map(async (item) => {
- const cache = await ctx.cache.get(item.link);
- if (cache) {
- return JSON.parse(cache);
- }
- const description =
- scheme === 'https' || !cdn
- ? item['content:encoded']
- : item['content:encoded'].replaceAll(/(?<=)/g, (match, p) => {
- if (p[0] === '/') {
- return cdn + feed.link + p;
- } else if (p.slice(0, 5) === 'http:') {
- return cdn + p;
- } else {
- return p;
- }
- });
- const article = {
- title: item.title,
- description,
- pubDate: item.pubDate,
- link: item.link,
- author: item.creator,
- };
- return article;
- })
- );
-
- ctx.state.data = {
- title: feed.title,
- link: feed.link,
- description: feed.description,
- item: items,
- };
-};
diff --git a/lib/routes/wordpress/index.ts b/lib/routes/wordpress/index.ts
new file mode 100644
index 000000000..05f0e4e0f
--- /dev/null
+++ b/lib/routes/wordpress/index.ts
@@ -0,0 +1,172 @@
+import { Route } from '@/types';
+
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import parser from '@/utils/rss-parser';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+
+import { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr } from './util';
+
+async function handler(ctx) {
+ const { url = 'https://wordpress.org/news', filter } = ctx.req.param();
+ const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50;
+
+ if (!config.feature.allow_user_supply_unsafe_domain) {
+ throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+ }
+
+ if (!/^(https?):\/\/[^\s#$./?].\S*$/i.test(url)) {
+ throw new Error('Invalid URL');
+ }
+
+ const cdn = config.wordpress.cdnUrl;
+ const rootUrl = url;
+
+ const filters = parseFilterStr(filter);
+ const filtersWithPair = await bakeFiltersWithPair(filters, rootUrl);
+
+ const searchParams = bakeFilterSearchParams(filters, 'name', false);
+ const apiSearchParams = bakeFilterSearchParams(filtersWithPair, 'id', true);
+
+ apiSearchParams.append('_embed', 'true');
+ apiSearchParams.append('per_page', String(limit));
+
+ const apiUrl = bakeUrl(`${apiSlug}/posts`, rootUrl, apiSearchParams);
+ const currentUrl = bakeUrl(getFilterParamsForUrl(filtersWithPair) ?? '', rootUrl, searchParams);
+
+ try {
+ const { data: response } = await got(apiUrl);
+
+ const items = (Array.isArray(response) ? response : JSON.parse(response.match(/(\[.*])$/)[1])).slice(0, limit).map((item) => {
+ const terminologies = item._embedded['wp:term'];
+ const guid = item.guid?.rendered ?? item.guid;
+
+ const $$ = load(item.content?.rendered ?? item.content);
+
+ $$('img').each((_, el) => {
+ el = $$(el);
+
+ const src = el.prop('src');
+
+ if (src.startsWith('/')) {
+ el.prop('src', `${cdn}${item.link}${src}`);
+ } else if (src.startsWith('http:')) {
+ el.prop('src', `${cdn}${src}`);
+ }
+ });
+
+ const description = $$.html();
+
+ return {
+ title: item.title?.rendered ?? item.title,
+ description,
+ pubDate: parseDate(item.date_gmt),
+ link: item.link,
+ category: [...new Set(terminologies.flat().map((c) => c.name))],
+ author: item._embedded.author.map((a) => a.name).join('/'),
+ guid,
+ id: guid,
+ content: {
+ html: description,
+ text: $$.text(),
+ },
+ updated: parseDate(item.modified_gmt),
+ };
+ });
+
+ const data = await fetchData(currentUrl, rootUrl);
+
+ return {
+ ...data,
+ item: items,
+ };
+ } catch {
+ const feed = await parser.parseURL(`${rootUrl}/feed/`);
+
+ const items = feed.items.map((item) => {
+ const guid = item.guid;
+
+ const $$ = load(item['content:encoded']);
+
+ $$('img').each((_, el) => {
+ el = $$(el);
+
+ const src = el.prop('src');
+
+ if (src.startsWith('/')) {
+ el.prop('src', `${cdn}${item.link}${src}`);
+ } else if (src.startsWith('http:')) {
+ el.prop('src', `${cdn}${src}`);
+ }
+ });
+
+ const description = $$.html();
+
+ return {
+ title: item.title,
+ description,
+ pubDate: parseDate(item.pubDate ?? ''),
+ link: item.link,
+ category: item.categories,
+ author: item.creator,
+ guid,
+ id: guid,
+ content: {
+ html: description,
+ text: $$.text(),
+ },
+ };
+ });
+
+ return {
+ title: feed.title,
+ description: feed.description,
+ link: feed.link,
+ item: items,
+ allowEmpty: true,
+ image: feed.image?.url,
+ language: feed.language,
+ };
+ }
+}
+
+export const route: Route = {
+ path: '/:url?/:filter{.+}?',
+ name: 'WordPress',
+ url: 'wordpress.org',
+ maintainers: ['nczitzk'],
+ handler,
+ example: '/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast',
+ parameters: { url: 'URL, by default', filter: 'Filter, see below' },
+ description: `If you subscribe to [WordPress News](https://wordpress.org/news/)，where the URL is \`https://wordpress.org/news/\`, Encode the URL using \`encodeURIComponent()\` and then use it as the parameter. Therefore, the route will be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews).
+
+ :::tip
+ If you wish to subscribe to specific categories or tags, you can fill in the "filter" parameter in the route. \`/category/Podcast\` to subscribe to the Podcast category. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast).
+
+ You can also subscribe to multiple categories. \`/category/Podcast,Community\` to subscribe to both the Podcast and Community categories. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community).
+
+ Categories and tags can be combined as well. \`/category/Releases/tag/tagging\` to subscribe to the Releases category and the tagging tag. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging).
+
+ You can also search for keywords. \`/search/Blog\` to search for the keyword "Blog". In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog).
+ :::`,
+ categories: ['blog'],
+
+ features: {
+ requireConfig: [
+ {
+ name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN',
+ description: `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`,
+ optional: false,
+ },
+ ],
+ requirePuppeteer: false,
+ antiCrawler: false,
+ supportRadar: false,
+ supportBT: false,
+ supportPodcast: false,
+ supportScihub: false,
+ },
+ radar: [],
+};
diff --git a/lib/routes/wordpress/namespace.ts b/lib/routes/wordpress/namespace.ts
new file mode 100644
index 000000000..9711e33af
--- /dev/null
+++ b/lib/routes/wordpress/namespace.ts
@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+ name: 'WordPress',
+ url: 'wordpress.org',
+ categories: ['blog'],
+ description: '',
+};
diff --git a/lib/routes/wordpress/util.ts b/lib/routes/wordpress/util.ts
new file mode 100644
index 000000000..b7152a536
--- /dev/null
+++ b/lib/routes/wordpress/util.ts
@@ -0,0 +1,307 @@
+import got from '@/utils/got';
+import { load } from 'cheerio';
+
+const apiSlug = 'wp-json/wp/v2';
+
+interface Filter {
+ id: string;
+ name: string;
+ slug: string;
+}
+
+const filterKeys: Record = {
+ search: 's',
+};
+
+const filterApiKeys: Record = {
+ category: 'categories',
+ tag: 'tags',
+ search: undefined,
+};
+
+const filterApiKeysWithNoId = new Set(['search']);
+
+/**
+ * Bake filter search parameters.
+ *
+ * @param filterPairs - The filter pairs object.
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ * @param pairKey - The filter pair key.
+ * e.g. `{ id: ..., name: ..., slug: ... }`.
+ * @param isApi - Indicates if the search parameters are for API.
+ * @returns The baked filter search parameters.
+ */
+const bakeFilterSearchParams = (filterPairs: Record, pairKey: string, isApi: boolean = false): URLSearchParams => {
+ /**
+ * Bake filters recursively.
+ *
+ * @param filterPairs - The filter pairs object.
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ * @param filterSearchParams - The filter search parameters.
+ * e.g. `category=a,b&tag=c`.
+ * @returns The baked filter search parameters.
+ * e.g. `category=a,b&tag=c`.
+ */
+ const bakeFilters = (filterPairs: Record, filterSearchParams: URLSearchParams): URLSearchParams => {
+ const keys = Object.keys(filterPairs).filter((key) => filterPairs[key]?.length > 0 && (isApi ? Object.hasOwn(filterApiKeys, key) : Object.hasOwn(filterKeys, key)));
+
+ if (keys.length === 0) {
+ return filterSearchParams;
+ }
+
+ const key = keys[0];
+ const pairs = filterPairs[key];
+
+ const originalFilters = { ...filterPairs };
+ delete originalFilters[key];
+
+ const filterKey = getFilterKeyForSearchParams(key, isApi);
+ const pairValues = pairs.map((pair) => (Object.hasOwn(pair, pairKey) ? pair[pairKey] : pair));
+
+ if (filterKey) {
+ filterSearchParams.append(filterKey, pairValues.join(','));
+ }
+
+ return bakeFilters(originalFilters, filterSearchParams);
+ };
+
+ return bakeFilters(filterPairs, new URLSearchParams());
+};
+
+/**
+ * Bake filters with pair.
+ *
+ * @param filters - The filters object.
+ * e.g. `{ category: [ a, b ], tag: [ c ] }`.
+ * @returns The baked filters.
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ */
+const bakeFiltersWithPair = async (filters: Record, rootUrl: string) => {
+ /**
+ * Bake keywords recursively.
+ *
+ * @param key - The key.
+ * e.g. `category` or `tag`.
+ * @param keywords - The keywords.
+ * e.g. `[ a, b ]`.
+ * @returns The baked keywords.
+ * e.g. `[ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ]`.
+ */
+ const bakeKeywords = async (key: string, keywords: string[]) => {
+ if (keywords.length === 0) {
+ return [];
+ }
+
+ const [keyword, ...rest] = keywords;
+
+ const filter = await getFilterByKeyAndKeyword(key, keyword, rootUrl);
+
+ return [
+ ...(filter?.id && filter?.slug
+ ? [
+ {
+ id: filter.id,
+ name: filter.name,
+ slug: filter.slug,
+ },
+ ]
+ : []),
+ ...(await bakeKeywords(key, rest)),
+ ];
+ };
+
+ /**
+ * Bake filters recursively.
+ *
+ * @param filters - The filters object.
+ * e.g. `{ category: [ a, b ], tag: [ c ] }`.
+ * @param filtersWithPair - The filters with pairs.
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ * @returns The baked filters.
+ * e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
+ */
+ const bakeFilters = async (filters: Record, filtersWithPair: Record) => {
+ const keys = Object.keys(filters);
+
+ if (keys.length === 0) {
+ return filtersWithPair;
+ }
+
+ const key = keys[0];
+ const keywords = filters[key];
+
+ const originalFilters = { ...filters };
+ delete originalFilters[key];
+
+ return bakeFilters(originalFilters, {
+ ...filtersWithPair,
+ [key]: filterApiKeysWithNoId.has(key) ? keywords : await bakeKeywords(key, keywords),
+ });
+ };
+
+ return await bakeFilters(filters, {});
+};
+
+/**
+ * Bake URL with search parameters.
+ *
+ * @param url - The URL.
+ * @param rootUrl - The root URL.
+ * @param searchParams - The search parameters.
+ * @returns The baked URL.
+ */
+const bakeUrl = (url: string, rootUrl: string, searchParams: URLSearchParams = new URLSearchParams()): string => {
+ const searchParamsStr = searchParams.toString();
+ const searchParamsSuffix = searchParamsStr ? `?${searchParamsStr}` : '';
+
+ return `${rootUrl}/${url}${searchParamsSuffix}`;
+};
+
+/**
+ * Fetch data from the specified URL.
+ *
+ * @param url - The URL to fetch data from.
+ * @param rootUrl - The root URL.
+ * @returns A promise that resolves to an object containing the fetched data to be added into `ctx.state.data`.
+ */
+const fetchData = async (url: string, rootUrl: string): Promise