feat(route): add WordPress (#15171)
* feat(route): add WordPress * fix docs * feat: add search keywords
This commit is contained in:
Ethan Shen
GitHub
parent
bec0e6d896
commit
4655e0952b
|
|
@ -1,50 +0,0 @@
|
|||
const parser = require('@/utils/rss-parser');
|
||||
const config = require('@/config').value;
|
||||
const allowDomain = new Set(['lawrence.code.blog']);
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.has(ctx.params.domain)) {
|
||||
ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
|
||||
}
|
||||
|
||||
const scheme = ctx.params.https || 'https';
|
||||
const cdn = config.wordpress.cdnUrl;
|
||||
|
||||
const domain = `${scheme}://${ctx.params.domain}`;
|
||||
const feed = await parser.parseURL(`${domain}/feed/`);
|
||||
const items = await Promise.all(
|
||||
feed.items.map(async (item) => {
|
||||
const cache = await ctx.cache.get(item.link);
|
||||
if (cache) {
|
||||
return JSON.parse(cache);
|
||||
}
|
||||
const description =
|
||||
scheme === 'https' || !cdn
|
||||
? item['content:encoded']
|
||||
: item['content:encoded'].replaceAll(/(?<=<img.*src=")(.*)(?=".*\/>)/g, (match, p) => {
|
||||
if (p[0] === '/') {
|
||||
return cdn + feed.link + p;
|
||||
} else if (p.slice(0, 5) === 'http:') {
|
||||
return cdn + p;
|
||||
} else {
|
||||
return p;
|
||||
}
|
||||
});
|
||||
const article = {
|
||||
title: item.title,
|
||||
description,
|
||||
pubDate: item.pubDate,
|
||||
link: item.link,
|
||||
author: item.creator,
|
||||
};
|
||||
return article;
|
||||
})
|
||||
);
|
||||
|
||||
ctx.state.data = {
|
||||
title: feed.title,
|
||||
link: feed.link,
|
||||
description: feed.description,
|
||||
item: items,
|
||||
};
|
||||
};
|
||||
|
|
@ -0,0 +1,172 @@
|
|||
import { Route } from '@/types';
|
||||
|
||||
import got from '@/utils/got';
|
||||
import { load } from 'cheerio';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import parser from '@/utils/rss-parser';
|
||||
import { config } from '@/config';
|
||||
import ConfigNotFoundError from '@/errors/types/config-not-found';
|
||||
|
||||
import { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr } from './util';
|
||||
|
||||
async function handler(ctx) {
|
||||
const { url = 'https://wordpress.org/news', filter } = ctx.req.param();
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50;
|
||||
|
||||
if (!config.feature.allow_user_supply_unsafe_domain) {
|
||||
throw new ConfigNotFoundError(`This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
|
||||
}
|
||||
|
||||
if (!/^(https?):\/\/[^\s#$./?].\S*$/i.test(url)) {
|
||||
throw new Error('Invalid URL');
|
||||
}
|
||||
|
||||
const cdn = config.wordpress.cdnUrl;
|
||||
const rootUrl = url;
|
||||
|
||||
const filters = parseFilterStr(filter);
|
||||
const filtersWithPair = await bakeFiltersWithPair(filters, rootUrl);
|
||||
|
||||
const searchParams = bakeFilterSearchParams(filters, 'name', false);
|
||||
const apiSearchParams = bakeFilterSearchParams(filtersWithPair, 'id', true);
|
||||
|
||||
apiSearchParams.append('_embed', 'true');
|
||||
apiSearchParams.append('per_page', String(limit));
|
||||
|
||||
const apiUrl = bakeUrl(`${apiSlug}/posts`, rootUrl, apiSearchParams);
|
||||
const currentUrl = bakeUrl(getFilterParamsForUrl(filtersWithPair) ?? '', rootUrl, searchParams);
|
||||
|
||||
try {
|
||||
const { data: response } = await got(apiUrl);
|
||||
|
||||
const items = (Array.isArray(response) ? response : JSON.parse(response.match(/(\[.*])$/)[1])).slice(0, limit).map((item) => {
|
||||
const terminologies = item._embedded['wp:term'];
|
||||
const guid = item.guid?.rendered ?? item.guid;
|
||||
|
||||
const $$ = load(item.content?.rendered ?? item.content);
|
||||
|
||||
$$('img').each((_, el) => {
|
||||
el = $$(el);
|
||||
|
||||
const src = el.prop('src');
|
||||
|
||||
if (src.startsWith('/')) {
|
||||
el.prop('src', `${cdn}${item.link}${src}`);
|
||||
} else if (src.startsWith('http:')) {
|
||||
el.prop('src', `${cdn}${src}`);
|
||||
}
|
||||
});
|
||||
|
||||
const description = $$.html();
|
||||
|
||||
return {
|
||||
title: item.title?.rendered ?? item.title,
|
||||
description,
|
||||
pubDate: parseDate(item.date_gmt),
|
||||
link: item.link,
|
||||
category: [...new Set(terminologies.flat().map((c) => c.name))],
|
||||
author: item._embedded.author.map((a) => a.name).join('/'),
|
||||
guid,
|
||||
id: guid,
|
||||
content: {
|
||||
html: description,
|
||||
text: $$.text(),
|
||||
},
|
||||
updated: parseDate(item.modified_gmt),
|
||||
};
|
||||
});
|
||||
|
||||
const data = await fetchData(currentUrl, rootUrl);
|
||||
|
||||
return {
|
||||
...data,
|
||||
item: items,
|
||||
};
|
||||
} catch {
|
||||
const feed = await parser.parseURL(`${rootUrl}/feed/`);
|
||||
|
||||
const items = feed.items.map((item) => {
|
||||
const guid = item.guid;
|
||||
|
||||
const $$ = load(item['content:encoded']);
|
||||
|
||||
$$('img').each((_, el) => {
|
||||
el = $$(el);
|
||||
|
||||
const src = el.prop('src');
|
||||
|
||||
if (src.startsWith('/')) {
|
||||
el.prop('src', `${cdn}${item.link}${src}`);
|
||||
} else if (src.startsWith('http:')) {
|
||||
el.prop('src', `${cdn}${src}`);
|
||||
}
|
||||
});
|
||||
|
||||
const description = $$.html();
|
||||
|
||||
return {
|
||||
title: item.title,
|
||||
description,
|
||||
pubDate: parseDate(item.pubDate ?? ''),
|
||||
link: item.link,
|
||||
category: item.categories,
|
||||
author: item.creator,
|
||||
guid,
|
||||
id: guid,
|
||||
content: {
|
||||
html: description,
|
||||
text: $$.text(),
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
title: feed.title,
|
||||
description: feed.description,
|
||||
link: feed.link,
|
||||
item: items,
|
||||
allowEmpty: true,
|
||||
image: feed.image?.url,
|
||||
language: feed.language,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export const route: Route = {
|
||||
path: '/:url?/:filter{.+}?',
|
||||
name: 'WordPress',
|
||||
url: 'wordpress.org',
|
||||
maintainers: ['nczitzk'],
|
||||
handler,
|
||||
example: '/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast',
|
||||
parameters: { url: 'URL, <https://wordpress.org/news> by default', filter: 'Filter, see below' },
|
||||
description: `If you subscribe to [WordPress News](https://wordpress.org/news/)，where the URL is \`https://wordpress.org/news/\`, Encode the URL using \`encodeURIComponent()\` and then use it as the parameter. Therefore, the route will be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews).
|
||||
|
||||
:::tip
|
||||
If you wish to subscribe to specific categories or tags, you can fill in the "filter" parameter in the route. \`/category/Podcast\` to subscribe to the Podcast category. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast).
|
||||
|
||||
You can also subscribe to multiple categories. \`/category/Podcast,Community\` to subscribe to both the Podcast and Community categories. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Podcast,Community).
|
||||
|
||||
Categories and tags can be combined as well. \`/category/Releases/tag/tagging\` to subscribe to the Releases category and the tagging tag. In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/category/Releases/tag/tagging).
|
||||
|
||||
You can also search for keywords. \`/search/Blog\` to search for the keyword "Blog". In this case, the route would be [\`/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog\`](https://rsshub.app/wordpress/https%3A%2F%2Fwordpress.org%2Fnews/search/Blog).
|
||||
:::`,
|
||||
categories: ['blog'],
|
||||
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN',
|
||||
description: `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`,
|
||||
optional: false,
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportRadar: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [],
|
||||
};
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
import type { Namespace } from '@/types';
|
||||
|
||||
export const namespace: Namespace = {
|
||||
name: 'WordPress',
|
||||
url: 'wordpress.org',
|
||||
categories: ['blog'],
|
||||
description: '',
|
||||
};
|
||||
|
|
@ -0,0 +1,307 @@
|
|||
import got from '@/utils/got';
|
||||
import { load } from 'cheerio';
|
||||
|
||||
const apiSlug = 'wp-json/wp/v2';
|
||||
|
||||
interface Filter {
|
||||
id: string;
|
||||
name: string;
|
||||
slug: string;
|
||||
}
|
||||
|
||||
const filterKeys: Record<string, string> = {
|
||||
search: 's',
|
||||
};
|
||||
|
||||
const filterApiKeys: Record<string, string | undefined> = {
|
||||
category: 'categories',
|
||||
tag: 'tags',
|
||||
search: undefined,
|
||||
};
|
||||
|
||||
const filterApiKeysWithNoId = new Set(['search']);
|
||||
|
||||
/**
|
||||
* Bake filter search parameters.
|
||||
*
|
||||
* @param filterPairs - The filter pairs object.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
* @param pairKey - The filter pair key.
|
||||
* e.g. `{ id: ..., name: ..., slug: ... }`.
|
||||
* @param isApi - Indicates if the search parameters are for API.
|
||||
* @returns The baked filter search parameters.
|
||||
*/
|
||||
const bakeFilterSearchParams = (filterPairs: Record<string, Filter[] | string[]>, pairKey: string, isApi: boolean = false): URLSearchParams => {
|
||||
/**
|
||||
* Bake filters recursively.
|
||||
*
|
||||
* @param filterPairs - The filter pairs object.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
* @param filterSearchParams - The filter search parameters.
|
||||
* e.g. `category=a,b&tag=c`.
|
||||
* @returns The baked filter search parameters.
|
||||
* e.g. `category=a,b&tag=c`.
|
||||
*/
|
||||
const bakeFilters = (filterPairs: Record<string, Filter[] | string[]>, filterSearchParams: URLSearchParams): URLSearchParams => {
|
||||
const keys = Object.keys(filterPairs).filter((key) => filterPairs[key]?.length > 0 && (isApi ? Object.hasOwn(filterApiKeys, key) : Object.hasOwn(filterKeys, key)));
|
||||
|
||||
if (keys.length === 0) {
|
||||
return filterSearchParams;
|
||||
}
|
||||
|
||||
const key = keys[0];
|
||||
const pairs = filterPairs[key];
|
||||
|
||||
const originalFilters = { ...filterPairs };
|
||||
delete originalFilters[key];
|
||||
|
||||
const filterKey = getFilterKeyForSearchParams(key, isApi);
|
||||
const pairValues = pairs.map((pair) => (Object.hasOwn(pair, pairKey) ? pair[pairKey] : pair));
|
||||
|
||||
if (filterKey) {
|
||||
filterSearchParams.append(filterKey, pairValues.join(','));
|
||||
}
|
||||
|
||||
return bakeFilters(originalFilters, filterSearchParams);
|
||||
};
|
||||
|
||||
return bakeFilters(filterPairs, new URLSearchParams());
|
||||
};
|
||||
|
||||
/**
|
||||
* Bake filters with pair.
|
||||
*
|
||||
* @param filters - The filters object.
|
||||
* e.g. `{ category: [ a, b ], tag: [ c ] }`.
|
||||
* @returns The baked filters.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
*/
|
||||
const bakeFiltersWithPair = async (filters: Record<string, string[]>, rootUrl: string) => {
|
||||
/**
|
||||
* Bake keywords recursively.
|
||||
*
|
||||
* @param key - The key.
|
||||
* e.g. `category` or `tag`.
|
||||
* @param keywords - The keywords.
|
||||
* e.g. `[ a, b ]`.
|
||||
* @returns The baked keywords.
|
||||
* e.g. `[ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ]`.
|
||||
*/
|
||||
const bakeKeywords = async (key: string, keywords: string[]) => {
|
||||
if (keywords.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const [keyword, ...rest] = keywords;
|
||||
|
||||
const filter = await getFilterByKeyAndKeyword(key, keyword, rootUrl);
|
||||
|
||||
return [
|
||||
...(filter?.id && filter?.slug
|
||||
? [
|
||||
{
|
||||
id: filter.id,
|
||||
name: filter.name,
|
||||
slug: filter.slug,
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...(await bakeKeywords(key, rest)),
|
||||
];
|
||||
};
|
||||
|
||||
/**
|
||||
* Bake filters recursively.
|
||||
*
|
||||
* @param filters - The filters object.
|
||||
* e.g. `{ category: [ a, b ], tag: [ c ] }`.
|
||||
* @param filtersWithPair - The filters with pairs.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
* @returns The baked filters.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
*/
|
||||
const bakeFilters = async (filters: Record<string, string[]>, filtersWithPair: Record<string, Filter[]>) => {
|
||||
const keys = Object.keys(filters);
|
||||
|
||||
if (keys.length === 0) {
|
||||
return filtersWithPair;
|
||||
}
|
||||
|
||||
const key = keys[0];
|
||||
const keywords = filters[key];
|
||||
|
||||
const originalFilters = { ...filters };
|
||||
delete originalFilters[key];
|
||||
|
||||
return bakeFilters(originalFilters, {
|
||||
...filtersWithPair,
|
||||
[key]: filterApiKeysWithNoId.has(key) ? keywords : await bakeKeywords(key, keywords),
|
||||
});
|
||||
};
|
||||
|
||||
return await bakeFilters(filters, {});
|
||||
};
|
||||
|
||||
/**
|
||||
* Bake URL with search parameters.
|
||||
*
|
||||
* @param url - The URL.
|
||||
* @param rootUrl - The root URL.
|
||||
* @param searchParams - The search parameters.
|
||||
* @returns The baked URL.
|
||||
*/
|
||||
const bakeUrl = (url: string, rootUrl: string, searchParams: URLSearchParams = new URLSearchParams()): string => {
|
||||
const searchParamsStr = searchParams.toString();
|
||||
const searchParamsSuffix = searchParamsStr ? `?${searchParamsStr}` : '';
|
||||
|
||||
return `${rootUrl}/${url}${searchParamsSuffix}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Fetch data from the specified URL.
|
||||
*
|
||||
* @param url - The URL to fetch data from.
|
||||
* @param rootUrl - The root URL.
|
||||
* @returns A promise that resolves to an object containing the fetched data to be added into `ctx.state.data`.
|
||||
*/
|
||||
const fetchData = async (url: string, rootUrl: string): Promise<object> => {
|
||||
/**
|
||||
* Request URLs recursively.
|
||||
*
|
||||
* @param urls - The URLs to request.
|
||||
* @returns A promise that resolves to the response data or undefined if no response is available.
|
||||
*/
|
||||
const requestUrls = async (urls: string[]): Promise<string | undefined> => {
|
||||
if (urls.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const [currentUrl, ...remainingUrls] = urls;
|
||||
try {
|
||||
const { data: response } = await got.get(currentUrl);
|
||||
return response;
|
||||
} catch {
|
||||
return requestUrls(remainingUrls);
|
||||
}
|
||||
};
|
||||
|
||||
const response = await requestUrls([url, rootUrl]);
|
||||
|
||||
if (!response) {
|
||||
return {};
|
||||
}
|
||||
|
||||
const $ = load(response);
|
||||
|
||||
const title = $('title').first().text();
|
||||
const image = new URL('wp-content/uploads/site_logo.png', rootUrl).href;
|
||||
|
||||
return {
|
||||
title,
|
||||
description: $('meta[property="og:description"]').attr('content') || $('meta[name="description"]').attr('content'),
|
||||
link: url,
|
||||
allowEmpty: true,
|
||||
image,
|
||||
author: $('meta[property="og:site_name"]').attr('content'),
|
||||
language: $('html').attr('lang'),
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Get filter by key and keyword.
|
||||
*
|
||||
* @param key - The key.
|
||||
* e.g. `category` or `tag`.
|
||||
* @param keyword - The keywords.
|
||||
* e.g. `keyword1`.
|
||||
* @returns A promise that resolves to the filter object if found, or undefined if not found.
|
||||
*/
|
||||
const getFilterByKeyAndKeyword = async (key: string, keyword: string, rootUrl: string): Promise<Filter | undefined> => {
|
||||
const apiFilterUrl = `${rootUrl}/${apiSlug}/${getFilterKeyForSearchParams(key, true)}`;
|
||||
|
||||
const { data: response } = await got(apiFilterUrl, {
|
||||
searchParams: {
|
||||
search: keyword,
|
||||
},
|
||||
});
|
||||
|
||||
return response.length > 0 ? response[0] : undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get filter key for search parameters.
|
||||
*
|
||||
* @param key - The key. e.g. `category` or `tag`.
|
||||
* @param isApi - Indicates whether the key is for the API.
|
||||
* @returns The filter key for search parameters, or undefined if not found.
|
||||
* e.g. `categories` or `tags`.
|
||||
*/
|
||||
const getFilterKeyForSearchParams = (key: string, isApi: boolean = false): string | undefined => {
|
||||
const keys = isApi ? filterApiKeys : filterKeys;
|
||||
|
||||
return Object.hasOwn(keys, key) ? keys[key] ?? key : undefined;
|
||||
};
|
||||
|
||||
/**
|
||||
* Get filter parameters for URL.
|
||||
*
|
||||
* @param filterPairs - The filter pairs object.
|
||||
* e.g. `{ category: [ { id: ..., name: ..., slug: ... }, { id: ..., name: ..., slug: ... } ], tag: [ { id: ..., name: ..., slug: ... } ] }`.
|
||||
* @returns The filter parameters for the URL, or undefined if no filters are available.
|
||||
*/
|
||||
const getFilterParamsForUrl = (filterPairs: Record<string, Filter[]>): string | undefined => {
|
||||
const keys = Object.keys(filterPairs).filter((key) => filterPairs[key].length > 0 && !Object.hasOwn(filterKeys, key));
|
||||
|
||||
if (keys.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const key = keys[0];
|
||||
|
||||
return `${key}/${filterPairs[key].map((pair) => pair.slug).join('/')}`;
|
||||
};
|
||||
|
||||
/**
|
||||
* Parses a filter string into a filters object.
|
||||
*
|
||||
* @param filterStr - The filter string to parse.
|
||||
* e.g. `category/a,b/tag/c`.
|
||||
* @returns The parsed filters object.
|
||||
* e.g. `{ category: [ 'a', 'b' ], tag: [ 'c' ] }`.
|
||||
*/
|
||||
const parseFilterStr = (filterStr: string | undefined): Record<string, string[]> => {
|
||||
/**
|
||||
* Recursively parses a filter string.
|
||||
*
|
||||
* @param remainingStr - The remaining filter string to parse.
|
||||
* e.g. `category/a,b/tag/c`.
|
||||
* @param filters - The accumulated filters object.
|
||||
* e.g. `{ category: [ a, b ], tag: [ c ] }`.
|
||||
* @param currentKey - The current filter key.
|
||||
* e.g. `category` or `tag`.
|
||||
* @returns The parsed filters object.
|
||||
*/
|
||||
const parseStr = (remainingStr: string | undefined, filters: Record<string, string[]> = {}, currentKey?: string): Record<string, string[]> => {
|
||||
if (!remainingStr) {
|
||||
return filters;
|
||||
}
|
||||
|
||||
const [word, ...rest] = remainingStr.split(/\/|,/);
|
||||
|
||||
const isKey = Object.hasOwn(filterApiKeys, word);
|
||||
const key = isKey ? word : currentKey;
|
||||
|
||||
const newFilters = key
|
||||
? {
|
||||
...filters,
|
||||
[key]: [...(filters[key] || []), ...(isKey ? [] : [word])],
|
||||
}
|
||||
: filters;
|
||||
|
||||
return parseStr(rest.join('/'), newFilters, key);
|
||||
};
|
||||
|
||||
return parseStr(filterStr, {});
|
||||
};
|
||||
|
||||
export { apiSlug, bakeFilterSearchParams, bakeFiltersWithPair, bakeUrl, fetchData, getFilterParamsForUrl, parseFilterStr };
|
||||
Loading…
Reference in New Issue