diff --git a/lib/routes/0818tuan/index.ts b/lib/routes/0818tuan/index.ts index 39fcc7f56..6605b42ef 100644 --- a/lib/routes/0818tuan/index.ts +++ b/lib/routes/0818tuan/index.ts @@ -44,7 +44,7 @@ async function handler(ctx) { link: item.attr('href').startsWith('http') ? item.attr('href') : `${baseUrl}${item.attr('href')}`, }; }) - .filter((i) => !i.link.includes('m.0818tuan.com/tb1111.php')); + .filter((i) => !i.link.includes('m.0818tuan.com/tb1111.php') && !i.link.includes('www.0818tuan.com/pdd/zudui.php')); const items = await Promise.all( list.map((item) => diff --git a/lib/routes/agirls/topic-list.ts b/lib/routes/agirls/topic-list.ts index 5d4c48f5e..feeaac798 100644 --- a/lib/routes/agirls/topic-list.ts +++ b/lib/routes/agirls/topic-list.ts @@ -1,7 +1,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { baseUrl } from './utils'; @@ -33,9 +33,9 @@ async function handler() { const category = 'topic'; const link = `${baseUrl}/${category}`; - const response = await got(`${baseUrl}/${category}`); + const response = await ofetch(`${baseUrl}/${category}`); - const $ = load(response.data); + const $ = load(response); const items = $('.ag-topic') .toArray() diff --git a/lib/routes/agirls/topic.ts b/lib/routes/agirls/topic.ts index ff67a670c..d8c9ae56c 100644 --- a/lib/routes/agirls/topic.ts +++ b/lib/routes/agirls/topic.ts @@ -2,7 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { baseUrl, parseArticle } from './utils'; @@ -32,9 +32,9 @@ export const route: Route = { async function handler(ctx) { const topic = ctx.req.param('topic'); const link = `${baseUrl}/topic/${topic}`; - const response = await got(link); + const response = await ofetch(link); - const $ = load(response.data); + const $ = load(response); const ldJson = JSON.parse($('script[type="application/ld+json"]').text()); const list = $('.ag-post-item__link') .toArray() diff --git a/lib/routes/agirls/utils.ts b/lib/routes/agirls/utils.ts index 31d2d05a9..b1cafcc48 100644 --- a/lib/routes/agirls/utils.ts +++ b/lib/routes/agirls/utils.ts @@ -1,13 +1,13 @@ import { load } from 'cheerio'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; const baseUrl = 'https://agirls.aotter.net'; const parseArticle = async (item) => { - const detailResponse = await got(item.link); - const content = load(detailResponse.data); + const detailResponse = await ofetch(item.link); + const content = load(detailResponse); item.category = [ ...new Set( @@ -17,11 +17,12 @@ const parseArticle = async (item) => { ), ]; const ldJson = JSON.parse(content('script[type="application/ld+json"]').text()); + const newsArticle = ldJson['@graph'].find((g) => g['@type'] === 'NewsArticle'); item.description = content('.ag-article__content').html(); - item.pubDate = parseDate(ldJson['@graph'][0].datePublished); // 2023-07-05T12:11:36+08:00 - item.updated = parseDate(ldJson['@graph'][0].dateModified); // 2023-07-05T12:11:36+08:00 - item.author = ldJson['@graph'][0].author.map((a) => a.name).join(', '); + item.pubDate = parseDate(newsArticle.datePublished); // 2023-07-05T12:11:36+08:00 + item.updated = parseDate(newsArticle.dateModified); // 2023-07-05T12:11:36+08:00 + item.author = newsArticle.author.map((a) => a.name).join(', '); return item; }; diff --git a/lib/routes/agirls/z-index.ts b/lib/routes/agirls/z-index.ts index cfab56f9d..c1efd1a38 100644 --- a/lib/routes/agirls/z-index.ts +++ b/lib/routes/agirls/z-index.ts @@ -2,7 +2,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { baseUrl, parseArticle } from './utils'; @@ -36,11 +36,11 @@ export const route: Route = { async function handler(ctx) { const { category = '' } = ctx.req.param(); const link = `${baseUrl}/posts${category ? `/${category}` : ''}`; - const response = await got(link); + const response = await ofetch(link); - const $ = load(response.data); + const $ = load(response); - const list = $('.ag-post-item__link') + const list = $('.ag-post-list .ag-post-item__link') .toArray() .map((item) => { item = $(item); diff --git a/lib/routes/anthropic/engineering.ts b/lib/routes/anthropic/engineering.ts index 4fc6cdd56..c9528885f 100644 --- a/lib/routes/anthropic/engineering.ts +++ b/lib/routes/anthropic/engineering.ts @@ -4,6 +4,7 @@ import pMap from 'p-map'; import type { DataItem, Route } from '@/types'; import cache from '@/utils/cache'; import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; export const route: Route = { path: '/engineering', @@ -28,17 +29,17 @@ async function handler(ctx) { const $ = load(response); const limit = ctx.req.query('limit') ? Number(ctx.req.query('limit')) : 20; - const list: DataItem[] = $('a[class*="cardLink"]') + const list: DataItem[] = $('a[class$="cardLink"]') .toArray() .map((element) => { const $e = $(element); const href = $e.attr('href') ?? ''; const fullLink = href.startsWith('http') ? href : `${baseUrl}${href}`; - const pubDate = $e.find('div[class*="date"]').text(); + const dateText = $e.find('div[class$="date"]').text(); return { title: $e.find('h2, h3').text(), link: fullLink, - pubDate, + pubDate: dateText ? parseDate(dateText, 'MMM D, YYYY') : undefined, }; }) .filter((item) => item.title && item.link) @@ -51,7 +52,7 @@ async function handler(ctx) { const response = await ofetch(item.link!); const $ = load(response); - const content = $('article > div > div[class*="__body"]'); + const content = $('article > div > div[class$="__body"]'); content.find('img').each((_, e) => { const $e = $(e); @@ -65,6 +66,8 @@ async function handler(ctx) { }); item.description = content.html(); + const dateText = $('p[class$="date"]').text().replace('Published', '').trim(); + item.pubDate ||= dateText ? parseDate(dateText, 'MMM D, YYYY') : undefined; return item; }), diff --git a/lib/routes/byteclicks/index.ts b/lib/routes/byteclicks/index.ts index 67865b163..741758e67 100644 --- a/lib/routes/byteclicks/index.ts +++ b/lib/routes/byteclicks/index.ts @@ -1,38 +1,38 @@ +import { load } from 'cheerio'; +import pMap from 'p-map'; + import type { Route } from '@/types'; -import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; +import ofetch from '@/utils/ofetch'; -import { parseItem } from './utils'; - -const baseUrl = 'https://byteclicks.com'; +import { baseUrl, parseItem, parseList } from './utils'; export const route: Route = { path: '/', + categories: ['new-media'], + example: '/byteclicks', radar: [ { source: ['byteclicks.com/'], - target: '', }, ], - name: 'Unknown', + name: '首页', maintainers: ['TonyRL'], handler, url: 'byteclicks.com/', }; async function handler(ctx) { - const { data } = await got(`${baseUrl}/wp-json/wp/v2/posts`, { - searchParams: { - per_page: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 100, - }, - }); + const response = await ofetch(baseUrl, { headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME }); + const $ = load(response); - const items = parseItem(data); + const list = parseList($).slice(0, ctx.req.query('limit') ? Number(ctx.req.query('limit')) : undefined); + const items = await pMap(list, (item) => parseItem(item), { concurrency: 5 }); return { - title: '字节点击 - 聚合全球优质资源，跟踪世界前沿科技', - description: - 'byteclicks.com 最专业的前沿科技网站。聚合全球优质资源，跟踪世界前沿科技，精选推荐一些很棒的互联网好资源好工具好产品。寻找有前景好项目、找论文、找报告、找数据、找课程、找电子书上byteclicks！byteclicks.com是投资人、科研学者、学生每天必看的网站。', - image: 'https://byteclicks.com/wp-content/themes/RK-Blogger/images/wbolt.ico', + title: $('head title').text(), + description: $('head meta[name="description"]').attr('content'), + image: $('head link[rel="shortcut icon"]').attr('href'), link: baseUrl, item: items, }; diff --git a/lib/routes/byteclicks/tag.ts b/lib/routes/byteclicks/tag.ts index 8a15daa89..0364e3458 100644 --- a/lib/routes/byteclicks/tag.ts +++ b/lib/routes/byteclicks/tag.ts @@ -1,23 +1,17 @@ +import { load } from 'cheerio'; +import pMap from 'p-map'; + import type { Route } from '@/types'; -import got from '@/utils/got'; +import { PRESETS } from '@/utils/header-generator'; +import ofetch from '@/utils/ofetch'; -import { parseItem } from './utils'; - -const baseUrl = 'https://byteclicks.com'; +import { baseUrl, parseItem, parseList } from './utils'; export const route: Route = { path: '/tag/:tag', categories: ['new-media'], example: '/byteclicks/tag/人工智能', parameters: { tag: '标签，可在URL中找到' }, - features: { - requireConfig: false, - requirePuppeteer: false, - antiCrawler: false, - supportBT: false, - supportPodcast: false, - supportScihub: false, - }, radar: [ { source: ['byteclicks.com/tag/:tag'], @@ -31,27 +25,18 @@ export const route: Route = { async function handler(ctx) { const tag = ctx.req.param('tag'); - const { data: search } = await got(`${baseUrl}/wp-json/wp/v2/tags`, { - searchParams: { - search: tag, - per_page: 100, - }, - }); - const tagData = search.find((item) => item.name === tag); + const link = `${baseUrl}/tag/${tag}`; - const { data } = await got(`${baseUrl}/wp-json/wp/v2/posts`, { - searchParams: { - per_page: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 100, - tags: tagData.id, - }, - }); + const response = await ofetch(link, { headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME }); + const $ = load(response); - const items = parseItem(data); + const list = parseList($).slice(0, ctx.req.query('limit') ? Number(ctx.req.query('limit')) : undefined); + const items = await pMap(list, (item) => parseItem(item), { concurrency: 5 }); return { - title: `${tagData.name} - 字节点击`, - image: 'https://byteclicks.com/wp-content/themes/RK-Blogger/images/wbolt.ico', - link: tagData.link, + title: $('head title').text(), + image: $('head link[rel="shortcut icon"]').attr('href'), + link, item: items, }; } diff --git a/lib/routes/byteclicks/utils.ts b/lib/routes/byteclicks/utils.ts index d8073f39e..11a48930e 100644 --- a/lib/routes/byteclicks/utils.ts +++ b/lib/routes/byteclicks/utils.ts @@ -1,11 +1,38 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; + +import type { DataItem } from '@/types'; +import cache from '@/utils/cache'; +import { PRESETS } from '@/utils/header-generator'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; +import timezone from '@/utils/timezone'; -const parseItem = (data) => - data.map((item) => ({ - title: item.title.rendered, - description: item.content.rendered, - pubDate: parseDate(item.date_gmt), - link: item.link, - })); +export const baseUrl = 'https://byteclicks.com'; -export { parseItem }; +export const parseList = ($: CheerioAPI) => + $('article.post') + .toArray() + .map((item) => { + const $item = $(item); + const a = $item.find('a.post-title'); + return { + title: a.text(), + link: a.attr('href'), + pubDate: timezone(parseDate($item.find('.meta-item.primary em').text(), 'YYYY.MM.DD'), 8), + category: $item.find('.cate-tag').text(), + }; + }); + +export const parseItem = (item: DataItem) => + cache.tryGet(item.link!, async () => { + const response = await ofetch(item.link!, { headerGeneratorOptions: PRESETS.MODERN_WINDOWS_CHROME }); + const $ = load(response); + + const content = $('.article-detail'); + content.find('.erphp-wppay, .copyright-message').remove(); + item.author = $('.meta-author em').text().trim(); + item.description = content.html()?.trim(); + + return item; + }); diff --git a/lib/routes/dcfever/reviews.ts b/lib/routes/dcfever/reviews.ts index 84e49498f..9baa959fb 100644 --- a/lib/routes/dcfever/reviews.ts +++ b/lib/routes/dcfever/reviews.ts @@ -39,7 +39,8 @@ async function handler(ctx) { title: item.text(), link: new URL(item.attr('href'), link).href, }; - }); + }) + .filter((item, index, arr) => arr.findIndex((i) => i.link === item.link) === index); const items = await Promise.all(list.map((item) => parseItem(item))); diff --git a/lib/routes/dcfever/utils.tsx b/lib/routes/dcfever/utils.tsx index 7045d8fa4..070606487 100644 --- a/lib/routes/dcfever/utils.tsx +++ b/lib/routes/dcfever/utils.tsx @@ -81,16 +81,16 @@ const parseItem = (item) => }); content.find('p a').each((_, e) => { - e = $(e); - if (e.text().startsWith('下一頁為')) { - e.remove(); + const $e = $(e); + if ($e.text().startsWith('下一頁為')) { + $e.remove(); } }); content.find('iframe').each((_, e) => { - e = $(e); - if (e.attr('src').startsWith('https://www.facebook.com/plugins/like.php')) { - e.remove(); + const $e = $(e); + if ($e.attr('src')?.startsWith('https://www.facebook.com/plugins/like.php')) { + $e.remove(); } }); diff --git a/lib/routes/hakkatv/type.ts b/lib/routes/hakkatv/news.ts similarity index 74% rename from lib/routes/hakkatv/type.ts rename to lib/routes/hakkatv/news.ts index f386fbb48..680d66816 100644 --- a/lib/routes/hakkatv/type.ts +++ b/lib/routes/hakkatv/news.ts @@ -1,6 +1,6 @@ import type { Route } from '@/types'; import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; @@ -41,41 +41,44 @@ async function handler(ctx) { const allData = type ? ( - await got(`${apiUrl}/api/news/index`, { - searchParams: { - per: 4, + await ofetch(`${apiUrl}/api/news/index`, { + query: { + per: 8, 'sort[created_at]': 'desc', type, keywords: '', }, }) - ).data.data + ).data : await Promise.all( typeMap.map(async (t) => { - const { data } = await got(`${apiUrl}/api/news/index`, { - searchParams: { - per: 4, + const { data } = await ofetch(`${apiUrl}/api/news/index`, { + query: { + per: 8, 'sort[created_at]': 'desc', type: t, keywords: '', }, }); - return data.data; + return data; }) ); - const list = allData.flat().map((item) => ({ - title: item.title, - pubDate: timezone(parseDate(item.created_at), 8), - author: item.author, - link: `${baseUrl}/news-detail/${item.id}`, - id: item.id, - })); + const list = allData + .flat() + .filter((item, index, arr) => arr.findIndex((i) => i.id === item.id) === index) + .map((item) => ({ + title: item.title, + pubDate: timezone(parseDate(item.created_at), 8), + author: item.author, + link: `${baseUrl}/news-detail/${item.id}`, + id: item.id, + })); const items = await Promise.all( list.map((item) => cache.tryGet(item.link, async () => { - const { data } = await got(`${apiUrl}/api/news/read/${item.id}`); + const data = await ofetch(`${apiUrl}/api/news/read/${item.id}`); item.category = data.tag.map((t) => t.tag); item.description = data.content.replaceAll('\n', '
'); delete item.id; diff --git a/lib/routes/immich/cursed-knowledge.ts b/lib/routes/immich/cursed-knowledge.ts index 93eaffe43..ec3e6e2d9 100644 --- a/lib/routes/immich/cursed-knowledge.ts +++ b/lib/routes/immich/cursed-knowledge.ts @@ -1,5 +1,3 @@ -import { load } from 'cheerio'; - import type { Route } from '@/types'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; @@ -19,31 +17,57 @@ export const route: Route = { handler, }; +const ghType = { + pr: 'pull', + issue: 'issues', + discussion: 'discussions', +}; + +const parseGithubLink = (arg: string) => { + const number = arg.match(/(?:number: )?([\d_]+)/)?.[1].replaceAll('_', ''); + const type = arg.match(/type: '(\w+)'/)?.[1] ?? 'pr'; + return `https://github.com/immich-app/immich/${ghType[type]}/${number}`; +}; + +const matchString = (entry: string, key: string) => { + const m = entry.match(new RegExp(`${key}:\\s*(?:'((?:[^'\\\\]|\\\\.)*)'|"((?:[^"\\\\]|\\\\.)*)")`)); + return (m?.[1] ?? m?.[2])?.replaceAll(/\\(.)/g, '$1'); +}; + async function handler() { const baseUrl = 'https://immich.app'; const link = `${baseUrl}/cursed-knowledge/`; - const response = await ofetch(link); - const $ = load(response); + const [source, feed] = await Promise.all([ + ofetch('https://raw.githubusercontent.com/immich-app/static-pages/main/apps/root.immich.app/src/routes/cursed-knowledge/+page.svelte'), + ofetch(`${baseUrl}/blog/feed.json`, { responseType: 'json' }), + ]); - const items = $('div.justify-around ul li') - .toArray() - .map((item) => { - const $item = $(item); - const href = $item.find('a').attr('href'); - const title = $item.find('section p').first().text(); - return { - title, - description: $item.find('section p').last().text(), - link: href ?? `${link}#${title}`, - pubDate: parseDate($item.find('div.justify-start').text()), - }; - }); + const entries = source + .slice(source.indexOf('const items')) + .split(/\n {4}(?:withBlog\()?\{\n/) + .slice(1); + + const items = entries.map((entry) => { + const blogId = entry.match(/id: '([^']+)'/)?.[1]; + const blogPost = blogId && feed.items.find((post) => post.id.endsWith(blogId)); + const gh = entry.match(/link: asGithubLink\(([^)]*)\)/); + const date = entry.match(/new Date\((\d+), (\d+), (\d+)\)/); + const title = matchString(entry, 'title'); + + return { + title, + description: matchString(entry, 'description'), + link: (gh ? parseGithubLink(gh[1]) : undefined) ?? entry.match(/href: '([^']+)'/)?.[1] ?? blogPost?.url, + guid: `${link}#${title}`, + pubDate: date ? parseDate(`${date[1]}-${Number(date[2]) + 1}-${date[3]}`, 'YYYY-M-D') : blogPost ? parseDate(blogPost.date_published) : undefined, + }; + }); return { - title: $('head title').text(), - description: $('p.text-center').text(), - image: `${baseUrl}${$('head link[rel="icon"]').attr('href')}`, + title: 'Cursed Knowledge | Immich', + description: 'Cursed knowledge we have learned as a result of building Immich that we wish we never knew.', + image: `${baseUrl}/favicon.ico`, link, item: items, }; diff --git a/lib/routes/indienova/gamedb.ts b/lib/routes/indienova/gamedb.ts index 34efc5a19..69a664b20 100644 --- a/lib/routes/indienova/gamedb.ts +++ b/lib/routes/indienova/gamedb.ts @@ -3,12 +3,26 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; import cache from '@/utils/cache'; import got from '@/utils/got'; -import { parseDate } from '@/utils/parse-date'; +import { parseDate, parseRelativeDate } from '@/utils/parse-date'; import timezone from '@/utils/timezone'; export const route: Route = { - path: '/gamedb/recent', - name: 'Unknown', + name: 'GameDB 游戏库', + path: '/gamedb/recent/:platform?', + example: '/indienova/gamedb/recent', + categories: ['game'], + parameters: { + platform: { + description: '平台，留空为 `all`', + options: [ + { value: 'all', label: '全部' }, + { value: 'ps4', label: 'PS4' }, + { value: 'xboxone', label: 'XBOX One' }, + { value: 'nintendo-switch', label: 'Nintendo Switch' }, + ], + default: 'all', + }, + }, maintainers: ['TonyRL'], handler, }; @@ -23,14 +37,15 @@ async function handler(ctx) { const list = $('.related-game') .toArray() .map((item) => { - item = $(item); + const $item = $(item); return { - title: item + title: $item .find('span') .contents() .filter((_, el) => el.nodeType === 3) .text(), - link: new URL(item.find('a').attr('href'), baseUrl).href, + link: new URL($item.find('a').attr('href')!, baseUrl).href, + pubDate: parseRelativeDate($item.find('small').first().text()), }; }); @@ -50,7 +65,7 @@ async function handler(ctx) { article.find('#showHiddenText').remove(); item.description = $('.cover-image').prop('outerHTML') + $('.tab-container').html() + article.html(); - item.pubDate = $('.gamedb-release').length ? timezone(parseDate($('.gamedb-release').text().replaceAll(/[()]/g, '')), 8) : null; + item.pubDate = $('.gamedb-release').length ? timezone(parseDate($('.gamedb-release').text().replaceAll(/[()]/g, '')), 8) : item.pubDate; return item; }) diff --git a/lib/routes/meteor/index.ts b/lib/routes/meteor/index.ts index 4ef131020..4243141cd 100644 --- a/lib/routes/meteor/index.ts +++ b/lib/routes/meteor/index.ts @@ -1,6 +1,5 @@ import type { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; import { baseUrl, getBoards, renderDesc } from './utils'; @@ -33,8 +32,9 @@ async function handler(ctx) { board = boardInfo.id; } - const { data: response } = await got.post(`${baseUrl}/article/get_new_articles`, { - json: { + const response = await ofetch(`${baseUrl}/article/get_new_articles`, { + method: 'POST', + body: { boardId: board, isCollege: false, page: 0, @@ -44,17 +44,14 @@ async function handler(ctx) { const result = JSON.parse(decodeURIComponent(response.result)); - const items = await Promise.all( - result.map((item) => - cache.tryGet(`meteor:${item.id}`, () => ({ - title: item.title, - description: renderDesc(item.content), - link: `${baseUrl}/article/${item.shortId}`, - author: item.authorAlias, - pubDate: parseDate(item.createdAt), - })) - ) - ); + const items = result.map((item) => ({ + title: item.title, + description: renderDesc(item.content), + link: `${baseUrl}/article/${item.shortId}`, + author: item.authorAlias, + pubDate: parseDate(item.createdAt), + category: item.tagNameList, + })); return { title: `${board === 'all' ? '全部看板' : boardInfo.title} | Meteor 學生社群`, diff --git a/lib/routes/meteor/utils.ts b/lib/routes/meteor/utils.ts index 15930b981..acee692be 100644 --- a/lib/routes/meteor/utils.ts +++ b/lib/routes/meteor/utils.ts @@ -1,5 +1,5 @@ import cache from '@/utils/cache'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { renderMedia } from './templates/desc'; @@ -7,28 +7,32 @@ const baseUrl = 'https://meteor.today'; const getBoards = () => cache.tryGet('meteor:boards', async () => { - const { data: response } = await got.post(`${baseUrl}/board/get_boards`, { - json: { + const response = await ofetch(`${baseUrl}/board/get_boards`, { + method: 'POST', + body: { isCollege: 'false', }, }); - return JSON.parse(decodeURIComponent(response.result)).map((item) => ({ - title: `${item.category ? `${item.category} - ` : ''}${item.name}`, - description: item.id, - feedDescription: item.description, - category: item.articleCategory, - link: `${baseUrl}/board/${item.alias ?? item.name}`, - alias: item.alias, - imgUrl: item.imageUrl, - id: item.id, - })); + return JSON.parse(decodeURIComponent(response.result)) + .map((item) => ({ + title: `${item.category ? `${item.category} - ` : ''}${item.name}`, + description: item.id, + feedDescription: item.description, + category: item.articleCategory, + link: `${baseUrl}/board/${item.alias ?? item.name}`, + alias: item.alias, + imgUrl: item.imageUrl, + id: item.id, + })) + .filter((item, index, arr) => arr.findIndex((i) => i.link === item.link) === index); }); const renderDesc = (desc) => { const youTube = /(?:https?:\/\/)?(?:www\.)?youtu\.?be.*(?:v=|v\/|\/)([\w-]+)&?/g; const matchYouTube = desc.match(youTube); const matchImgur = desc.match(/https:\/\/i.imgur.com\/\w*.(jpg|png|gif|jpeg)/g); + const matchImage = desc.match(/https:\/\/storage\.meteor\.today\/image\/[\da-f]{24}\.(jpg|png)/g); const matchVideo = desc.match(/(https:\/\/storage\.meteor\.today\/video\/[\da-f]{24}\.)(mp4|mov|avi|flv|wmv|mpeg|mkv)/gi); const matchSticker = desc.match(/assets\/images\/stickers\/(duck|ep2|ep1)\/\w*.(jpg|png|gif|jpeg)/g); const matchEmoji = desc.match(/assets\/images\/emoji\/\w*.(jpg|png|gif|jpeg)/g); @@ -49,6 +53,15 @@ const renderDesc = (desc) => { ); } } + if (matchImage) { + for (const img of matchImage) { + desc = desc.replace(img, () => + renderMedia({ + img, + }) + ); + } + } if (matchVideo) { for (const video of matchVideo) { desc = desc.replace(video, () => diff --git a/lib/routes/mhlw/monthly-labour-survey.ts b/lib/routes/mhlw/monthly-labour-survey.ts index b994e062e..4f7b0d9ea 100644 --- a/lib/routes/mhlw/monthly-labour-survey.ts +++ b/lib/routes/mhlw/monthly-labour-survey.ts @@ -22,6 +22,15 @@ export const route: Route = { url: 'www.mhlw.go.jp/toukei/list/30-1a.html', }; +const parseJapaneseDate = (text: string) => { + const normalized = text + .replaceAll(/（[^）]+）/g, '') + // oxlint-disable-next-line regexp/no-obscure-range + .replaceAll(/[０-９]/g, (c) => String.fromCodePoint(c.codePointAt(0)! - 0xfee0)) + .replace(/^令和(\d+)年/, (_, year) => `${Number(year) + 2018}年`); + return parseDate(normalized, 'YYYY年M月D日'); +}; + async function fetchPage(url: string) { const raw = await ofetch(url, { responseType: 'arrayBuffer' }); const decoder = new TextDecoder('shift-jis'); @@ -62,12 +71,11 @@ async function handler(ctx: Context) { const $ = load(response); const dateText = $('.prt-topContents .al-right').text(); - const cleanedDate = dateText.replaceAll(/（[^）]+）/g, ''); const content = $('#contentsInner'); content.find('.prt-topContents, .prt-linkNavi, .prt-plugin').remove(); - item.title = $('h1#pageTitle').text() || item.title; - item.pubDate = timezone(parseDate(cleanedDate, 'YYYY年M月D日'), 9); + item.title = $('h1#pageTitle').text().trim() || item.title; + item.pubDate = timezone(parseJapaneseDate(dateText), 9); item.description = content.html()?.trim(); return item; diff --git a/lib/routes/netflix/research.ts b/lib/routes/netflix/research.ts index 721de82c4..eba4a2970 100644 --- a/lib/routes/netflix/research.ts +++ b/lib/routes/netflix/research.ts @@ -1,6 +1,7 @@ import { load } from 'cheerio'; import type { Route } from '@/types'; +import { collapseWhitespace } from '@/utils/common-utils'; import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; @@ -72,6 +73,7 @@ async function handler() { title: item.title, description: item.description, link: item.link, + guid: collapseWhitespace(item.title) ?? undefined, pubDate: (item.date ?? item.startDate) ? parseDate(item.date ?? item.startDate) : undefined, category: item.tags?.json, image: item.image?.url, diff --git a/lib/routes/pubscholar/explore.ts b/lib/routes/pubscholar/explore.ts index cee31ba77..576ff8043 100644 --- a/lib/routes/pubscholar/explore.ts +++ b/lib/routes/pubscholar/explore.ts @@ -47,14 +47,17 @@ async function handler(ctx) { }, }); - const list = response.content.map((item) => ({ - title: (item.is_free || item.links.some((l) => l.is_open_access) ? '「Open Access」' : '') + sanitizeHtml(item.title, { allowedTags: [], allowedAttributes: {} }), - description: item.abstracts + `
${item.links.map((link) => `${link.is_open_access ? '「Open Access」' : ''}${link.name}`).join('
')}`, - author: item.author.join('; '), - pubDate: parseDate(item.date), - category: item.keywords.map((keyword) => sanitizeHtml(keyword, { allowedTags: [], allowedAttributes: {} })), - link: `${baseUrl}/${category}/${getArticleLink(item.id)}`, - })); + const list = response.content.map((item) => { + const date = item.date ?? item.issue_date ?? item.year; + return { + title: (item.is_free || item.links?.some((l) => l.is_open_access) ? '「Open Access」' : '') + sanitizeHtml(item.title, { allowedTags: [], allowedAttributes: {} }), + description: item.abstracts + `
${(item.links ?? []).map((link) => `${link.is_open_access ? '「Open Access」' : ''}${link.name}`).join('
')}`, + author: (item.author ?? item.inventors)?.join('; '), + pubDate: date ? parseDate(String(date), ['YYYY-MM-DD', 'YYYYMMDD', 'YYYY']) : undefined, + category: item.keywords?.map((keyword) => sanitizeHtml(keyword, { allowedTags: [], allowedAttributes: {} })), + link: `${baseUrl}/${category}/${getArticleLink(item.id)}`, + }; + }); return { title: 'PubScholar 公益学术平台', diff --git a/lib/routes/pubscholar/types.ts b/lib/routes/pubscholar/types.ts index 8d51adfbe..497ecb6fb 100644 --- a/lib/routes/pubscholar/types.ts +++ b/lib/routes/pubscholar/types.ts @@ -5,10 +5,10 @@ interface Link { } interface Content { - date: string; + date?: string; attachments: any[]; - keywords: string[]; - year: number; + keywords?: string[]; + year?: number | string; source: string; title: string; type: string; @@ -17,21 +17,23 @@ interface Content { school: any[]; first_page: string; local_links: any[]; - links: Link[]; + links?: Link[]; id: string; graduation_institution: any[]; cn_type: string; article_type: string; issue: string; abstracts: string; - author: string[]; + author?: string[]; + inventors?: string[]; + issue_date?: string; last_page: string; degree: string; tutor: any[]; semantic_entities: object; volume: string; source_list: string[]; - is_free: boolean; + is_free?: boolean; } export interface Resource { diff --git a/lib/routes/tingtingfm/program.tsx b/lib/routes/tingtingfm/program.tsx index 0ff2db14b..330b04159 100644 --- a/lib/routes/tingtingfm/program.tsx +++ b/lib/routes/tingtingfm/program.tsx @@ -50,7 +50,7 @@ async function handler(ctx) { const mobileBaseUrl = 'https://mobile.tingtingfm.com'; const params = { - version: 'h5_5.16', + version: 'h5_6.3.2', client: getClientVal(30), h_program_id: programId, }; diff --git a/lib/routes/tingtingfm/utils.ts b/lib/routes/tingtingfm/utils.ts index 02d8ec5b7..899d55999 100644 --- a/lib/routes/tingtingfm/utils.ts +++ b/lib/routes/tingtingfm/utils.ts @@ -1,20 +1,20 @@ -/* eslint-disable unicorn/prefer-code-point */ import md5 from '@/utils/md5'; const SALT = '1Ftjv0bfpVmqbE38'; +const randomChar = () => { + const random = Math.floor(62 * Math.random()); + if (random < 10) { + return random; + } + if (random < 36) { + return String.fromCodePoint(random + 55); + } + return String.fromCodePoint(random + 61); +}; + const getClientVal = (length) => { let result = ''; - const randomChar = () => { - const random = Math.floor(62 * Math.random()); - if (random < 10) { - return random; - } - if (random < 36) { - return String.fromCharCode(random + 55); - } - return String.fromCharCode(random + 61); - }; while (result.length < length) { result += randomChar(); }