fix(route): uber (#20034)
* fix route /uber/blog * refactor(route): Simplify JSON parsing and fix typo in description
This commit is contained in:
WilliamGates
GitHub
parent
f52dd2f06b
commit
35b9fa168e
|
|
@ -1,16 +1,17 @@
|
|||
import { Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import got from '@/utils/got';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { load } from 'cheerio';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const rootURL = 'https://www.uber.com';
|
||||
const apiURL = 'https://blogapi.uber.com';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/blog/:maxPage?',
|
||||
// `compat` is a never used parameter
|
||||
// just for backward compatibility with the deprecated `:maxPage` parameter
|
||||
path: '/blog/:compat?',
|
||||
categories: ['blog'],
|
||||
example: '/uber/blog',
|
||||
parameters: { maxPage: 'max number of pages to retrieve, default to 1 page at most' },
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
|
|
@ -21,65 +22,105 @@ export const route: Route = {
|
|||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['www.uber.com/:language/blog/engineering', 'www.uber.com/:language/blog'],
|
||||
source: ['www.uber.com/:language/blog/engineering'],
|
||||
target: '/blog',
|
||||
},
|
||||
],
|
||||
name: 'Engineering',
|
||||
maintainers: ['hulb'],
|
||||
handler,
|
||||
url: 'www.uber.com/blog/pittsburgh/engineering',
|
||||
url: 'www.uber.com/en-HK/blog/engineering',
|
||||
description:
|
||||
"The English blog on any of Uber's regional sites (e.g., www.uber.com/en-JP/blog) is the same engineering blog provided by this route, so language selection is not supported. This route is not for the public news blog on specific regional sites (e.g., www.uber.com/ja-JP/blog).",
|
||||
zh: {
|
||||
description: 'uber的任何区域站点的英文blog（例如www.uber.com/en-JP/blog）都是相同的内容，正是本路由提供的engineering blog，因此本路由不提供语言选择；本路由不是uber在特定区域站点的公开新闻blog（例如www.uber.com/ja-JP/blog)',
|
||||
},
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
let maxPage = Number(ctx.req.param('maxPage'));
|
||||
if (Number.isNaN(maxPage)) {
|
||||
maxPage = 1;
|
||||
}
|
||||
|
||||
let pages = await Promise.all(
|
||||
[...Array.from({ length: maxPage }).keys()].map((pageIdx) =>
|
||||
got(`${apiURL}/wp-json/blog/v1/data`, {
|
||||
searchParams: {
|
||||
page: pageIdx + 1,
|
||||
parent: 'pittsburgh',
|
||||
slug: 'engineering',
|
||||
},
|
||||
})
|
||||
)
|
||||
);
|
||||
pages = pages.map((page) => page.data);
|
||||
async function handler() {
|
||||
const response = await ofetch(`${rootURL}/en-HK/blog/engineering/rss/`, {
|
||||
// The source site is misconfigured or intentionally blocking requests without a specific accept header
|
||||
// Without this header, it will return an HTTP 406 error
|
||||
// Note that the accept type must be 'text/html'; 'application/xml' or similar will get HTTP 404 error
|
||||
headers: {
|
||||
accept: 'text/html',
|
||||
},
|
||||
// Without this, ofetch will parse the response as a blob instead of text, which cannot be loaded by cheerio
|
||||
parseResponse: (txt) => txt,
|
||||
});
|
||||
const $ = load(response, { xmlMode: true });
|
||||
|
||||
const result = await Promise.all(
|
||||
pages.map((page) =>
|
||||
Promise.all(
|
||||
page.posts.map((post) =>
|
||||
cache.tryGet(`${rootURL}${post.link}`, async () => {
|
||||
let { data: article } = await got(`${apiURL}/wp-json/blog/v1/data`, {
|
||||
searchParams: {
|
||||
slug: post.link.replaceAll('/', '').replace('blog', ''),
|
||||
},
|
||||
});
|
||||
article = article.article;
|
||||
$('item')
|
||||
.toArray()
|
||||
.map((el) =>
|
||||
cache.tryGet($(el).find('link').text(), async () => {
|
||||
const detailResponse = await ofetch($(el).find('link').text(), {
|
||||
headers: {
|
||||
accept: 'text/html',
|
||||
},
|
||||
});
|
||||
const detail = load(detailResponse);
|
||||
|
||||
return {
|
||||
link: article.link,
|
||||
title: article.title,
|
||||
description: article.content,
|
||||
pubDate: parseDate(article.created),
|
||||
author: article.author,
|
||||
category: article.categories.map((category) => category.category_name),
|
||||
};
|
||||
})
|
||||
)
|
||||
const scriptText = detail('script#__REDUX_STATE__').text().trim();
|
||||
// The json in the script element is over-encoded
|
||||
// It needs to be decoded this way before it can be parsed by JSON.parse
|
||||
const jsonText = decodeURIComponent(JSON.parse(`"${scriptText}"`));
|
||||
// Traverse the JSON to find the content node, which is more robust against format changes.
|
||||
const contentHtml = findNode(JSON.parse(jsonText), { idKey: 'id', idValue: 'BlogArticleContent', siblingKey: 'props', childKey: 'content' }).replaceAll(String.raw`\n`, '');
|
||||
|
||||
return {
|
||||
link: $(el).find('link').text(),
|
||||
title: $(el).find('title').text(),
|
||||
description: contentHtml,
|
||||
pubDate: parseDate($(el).find('pubDate').text()),
|
||||
category: $(el)
|
||||
.find('category')
|
||||
.toArray()
|
||||
.map((item) => $(item).text()),
|
||||
};
|
||||
})
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
title: `Uber Engineering Blog`,
|
||||
link: rootURL + '/blog/engineering',
|
||||
description: 'The technology behind Uber Engineering',
|
||||
item: result.flat(),
|
||||
item: result,
|
||||
};
|
||||
}
|
||||
|
||||
function findNode(
|
||||
json: any,
|
||||
options: {
|
||||
idKey?: string;
|
||||
idValue: string;
|
||||
siblingKey: string;
|
||||
childKey: string;
|
||||
}
|
||||
): any {
|
||||
const { idKey = 'id', idValue, siblingKey, childKey } = options;
|
||||
|
||||
if (Array.isArray(json)) {
|
||||
for (const item of json) {
|
||||
const result = findNode(item, options);
|
||||
if (result !== undefined) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
} else if (json && typeof json === 'object') {
|
||||
if (json[idKey] === idValue) {
|
||||
return json[siblingKey]?.[childKey];
|
||||
}
|
||||
|
||||
for (const key in json) {
|
||||
const result = findNode(json[key], options);
|
||||
if (result !== undefined) {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue