From 7ac4ffd1ae28753fa2b7f7a77564152da7a0c52f Mon Sep 17 00:00:00 2001 From: Jia Chen Date: Mon, 25 Mar 2024 00:27:11 +0800 Subject: [PATCH 1/7] feat(route): recover route blogread/newest (#14937) * feat(route): recover route blogread/newest * fix(route): Update lib/routes/blogread/index.ts * Update lib/routes/blogread/index.ts --------- --- lib/router.js | 3 -- .../newest.js => routes/blogread/index.ts} | 29 ++++++++++++++----- lib/routes/blogread/namespace.ts | 6 ++++ 3 files changed, 28 insertions(+), 10 deletions(-) rename lib/{routes-deprecated/blogread/newest.js => routes/blogread/index.ts} (52%) create mode 100644 lib/routes/blogread/namespace.ts diff --git a/lib/router.js b/lib/router.js index 6acc307b7..215bd0e56 100644 --- a/lib/router.js +++ b/lib/router.js @@ -379,9 +379,6 @@ router.get('/geekpark/breakingnews', lazyloadRouteHandler('./routes/geekpark/bre // 香港天文台 router.get('/hko/weather', lazyloadRouteHandler('./routes/hko/weather')); -// 技术头条 -router.get('/blogread/newest', lazyloadRouteHandler('./routes/blogread/newest')); - // gnn游戏新闻 router.get('/gnn/gnn', lazyloadRouteHandler('./routes/gnn/gnn')); diff --git a/lib/routes-deprecated/blogread/newest.js b/lib/routes/blogread/index.ts similarity index 52% rename from lib/routes-deprecated/blogread/newest.js rename to lib/routes/blogread/index.ts index 8e1872b2c..bef75385c 100644 --- a/lib/routes-deprecated/blogread/newest.js +++ b/lib/routes/blogread/index.ts @@ -1,8 +1,23 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); +import { Route } from '@/types'; +import got from '@/utils/got'; +import * as cheerio from 'cheerio'; -module.exports = async (ctx) => { - const url = 'http://blogread.cn/news/newest.php'; +export const route: Route = { + path: '/newest', + categories: ['programming'], + example: '/blogread/newest', + radar: [ + { + source: ['blogread.cn/news/newest.php'], + }, + ], + name: '最新文章', + maintainers: ['fashioncj'], + handler, +}; + +async function handler() { + const url = 'https://blogread.cn/news/newest.php'; const response = await got({ method: 'get', url, @@ -12,19 +27,19 @@ module.exports = async (ctx) => { .map((index, elem) => { elem = $(elem); const $link = elem.find('dt a'); - return { title: $link.text(), description: elem.find('dd').eq(0).text(), link: $link.attr('href'), author: elem.find('.small a').eq(0).text(), + pubDate: elem.find('dd').eq(1).text().split('\n')[2], }; }) .get(); - ctx.state.data = { + return { title: '技术头条', link: url, item: resultItem, }; -}; +} diff --git a/lib/routes/blogread/namespace.ts b/lib/routes/blogread/namespace.ts new file mode 100644 index 000000000..7f56723d3 --- /dev/null +++ b/lib/routes/blogread/namespace.ts @@ -0,0 +1,6 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: '技术头条', + url: 'blogread.cn', +}; From 0fea009f70e7782453c2c43bf35e12e20d260d92 Mon Sep 17 00:00:00 2001 From: Ethan Shen <42264778+nczitzk@users.noreply.github.com> Date: Mon, 25 Mar 2024 02:12:49 +0800 Subject: [PATCH 2/7] feat(route): add 1x.com (#14936) * feat(route): add 1x.com * fix radar * fix example --- lib/routes-deprecated/1x/index.js | 69 -------------- lib/routes/1x/index.ts | 121 ++++++++++++++++++++++++ lib/routes/1x/namespace.ts | 8 ++ lib/routes/1x/templates/description.art | 17 ++++ 4 files changed, 146 insertions(+), 69 deletions(-) delete mode 100644 lib/routes-deprecated/1x/index.js create mode 100644 lib/routes/1x/index.ts create mode 100644 lib/routes/1x/namespace.ts create mode 100644 lib/routes/1x/templates/description.art diff --git a/lib/routes-deprecated/1x/index.js b/lib/routes-deprecated/1x/index.js deleted file mode 100644 index fee500801..000000000 --- a/lib/routes-deprecated/1x/index.js +++ /dev/null @@ -1,69 +0,0 @@ -const got = require('@/utils/got'); -const cheerio = require('cheerio'); - -const categories = { - latest: 'latest', - popular: 'popular', - published: 'published', - abstract: 'latest:15:', - action: 'latest:1:', - animals: 'latest:21:', - architecture: 'latest:11:', - conceptual: 'latest:17:', - 'creative-edit': 'latest:10:', - documentary: 'latest:8:', - everyday: 'latest:14:', - 'fine-art-nude': 'latest:12:', - humour: 'latest:3:', - landscape: 'latest:6:', - macro: 'latest:2:', - mood: 'latest:4:', - night: 'latest:9:', - performance: 'latest:19:', - portrait: 'latest:13:', - 'still-life': 'latest:18:', - street: 'latest:7:', - underwater: 'latest:20:', - wildlife: 'latest:5:', -}; - -module.exports = async (ctx) => { - const category = ctx.params.category || 'latest'; - - const rootUrl = `https://1x.com`; - const currentUrl = `${rootUrl}/gallery/${category}`; - const apiUrl = `${rootUrl}/backend/lm.php?style=normal&mode=${categories[category]}&from=0&autoload=`; - const response = await got({ - method: 'get', - url: apiUrl, - }); - - const $ = cheerio.load(response.data); - - const items = $('root data') - .html() - .split('\n') - .slice(0, -1) - .map((item) => { - item = $(item); - - const id = item - .find('.photos-feed-image') - .attr('id') - .match(/img-(\d+)/)[1]; - - return { - guid: id, - link: `${rootUrl}/photo/${id}`, - author: item.find('.photos-feed-data-name').eq(0).text(), - title: item.find('.photos-feed-data-title').text() || 'Untitled', - description: ``, - }; - }); - - ctx.state.data = { - title: `${category} - 1X`, - link: currentUrl, - item: items, - }; -}; diff --git a/lib/routes/1x/index.ts b/lib/routes/1x/index.ts new file mode 100644 index 000000000..f978ff551 --- /dev/null +++ b/lib/routes/1x/index.ts @@ -0,0 +1,121 @@ +import { Route } from '@/types'; +import { getCurrentPath } from '@/utils/helpers'; +const __dirname = getCurrentPath(import.meta.url); + +import got from '@/utils/got'; +import { load } from 'cheerio'; +import { art } from '@/utils/render'; +import * as path from 'node:path'; + +export const handler = async (ctx) => { + const { category = 'latest/awarded' } = ctx.req.param(); + const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30; + + const rootUrl = 'https://1x.com'; + const currentUrl = new URL(`gallery/${category}`, rootUrl).href; + + const { data: currentResponse } = await got(currentUrl); + + const $ = load(currentResponse); + + const language = $('html').prop('lang'); + const apiUrl = new URL(`backend/lm2.php?style=normal&mode=${$('input#lm_mode').prop('value')}`, rootUrl).href; + + const { data: response } = await got(apiUrl); + + const $$ = load(response); + + const items = $$('div.photos-feed-item') + .slice(0, limit) + .toArray() + .map((item) => { + item = $(item); + + const title = item.find('span.photos-feed-data-title').first().text() || 'Untitled'; + const image = item.find('img').prop('src'); + const author = item.find('span.photos-feed-data-name').first().text(); + + const text = `${title} by ${author}`; + + const description = art(path.join(__dirname, 'templates/description.art'), { + images: image + ? [ + { + src: image, + alt: title, + }, + ] + : undefined, + description: text, + }); + + const id = item.find('img[id]').prop('id').split(/-/).pop(); + const guid = `1x-${id}`; + + return { + title, + description, + link: new URL(`photo/${id}`, rootUrl).href, + author, + guid, + id: guid, + content: { + html: description, + text, + }, + image, + banner: image, + language, + enclosure_url: image, + enclosure_type: image ? `image/${image.split(/\./).pop()}` : undefined, + enclosure_title: title, + }; + }); + + const image = new URL($('img.themedlogo').prop('src'), rootUrl).href; + + return { + title: $('title').text(), + description: $('meta[name="description"]').prop('content'), + link: currentUrl, + item: items, + allowEmpty: true, + image, + author: $('meta[property="og:site_name"]').prop('content'), + language, + }; +}; + +export const route: Route = { + path: '/:category{.+}?', + name: 'Gallery', + url: '1x.com', + maintainers: ['nczitzk'], + handler, + example: '/1x/latest/awarded', + parameters: { category: 'Category, Latest Awarded by default' }, + description: `::: tip +Fill in the field in the path with the part of the corresponding page URL after \`https://1x.com/gallery/\` or \`https://1x.com/photo/\`. Here are the examples: + +If you subscribe to [Abstract Awarded](https://1x.com/gallery/abstract/awarded), you should fill in the path with the part \`abstract/awarded\` from the page URL \`https://1x.com/gallery/abstract/awarded\`. In this case, the route will be [\`/1x/abstract/awarded\`](https://rsshub.app/1x/abstract/awarded). + +If you subscribe to [Wildlife Published](https://1x.com/gallery/wildlife/published), you should fill in the path with the part \`wildlife/published\` from the page URL \`https://1x.com/gallery/wildlife/published\`. In this case, the route will be [\`/1x/wildlife/published\`](https://rsshub.app/1x/wildlife/published). +:::`, + categories: ['design', 'picture'], + + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: true, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['/gallery/:category*', '/photos/:category*'], + target: '/1x/:category', + }, + ], +}; diff --git a/lib/routes/1x/namespace.ts b/lib/routes/1x/namespace.ts new file mode 100644 index 000000000..4e8cc283e --- /dev/null +++ b/lib/routes/1x/namespace.ts @@ -0,0 +1,8 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: '1x.com', + url: '1x.com', + categories: ['design', 'picture'], + description: '1x.com • In Pursuit of the Sublime. Browse 200,000 curated photos from photographers all over the world.', +}; diff --git a/lib/routes/1x/templates/description.art b/lib/routes/1x/templates/description.art new file mode 100644 index 000000000..dfab19230 --- /dev/null +++ b/lib/routes/1x/templates/description.art @@ -0,0 +1,17 @@ +{{ if images }} + {{ each images image }} + {{ if image?.src }} +
+ {{ image.alt }} +
+ {{ /if }} + {{ /each }} +{{ /if }} + +{{ if description }} + {{@ description }} +{{ /if }} \ No newline at end of file From a7053991e982ebf29d121ba3285a2f1ae55396be Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 25 Mar 2024 02:22:24 +0800 Subject: [PATCH 3/7] chore: update CI name --- .github/workflows/test-full-routes.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-full-routes.yml b/.github/workflows/test-full-routes.yml index 9e342975d..eddbf5aae 100644 --- a/.github/workflows/test-full-routes.yml +++ b/.github/workflows/test-full-routes.yml @@ -1,4 +1,4 @@ -name: Build assets +name: Build assets (Full Routes Test Result) on: workflow_dispatch: From 970ad011d049dfdc061cb02ec6e2138c2c591d44 Mon Sep 17 00:00:00 2001 From: Asuna Date: Mon, 25 Mar 2024 22:48:11 +0800 Subject: [PATCH 4/7] fix: update bilibili dynamic API to v2 from v1 (#14957) --- lib/routes/bilibili/dynamic.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/routes/bilibili/dynamic.ts b/lib/routes/bilibili/dynamic.ts index 8e127506e..7873ccb46 100644 --- a/lib/routes/bilibili/dynamic.ts +++ b/lib/routes/bilibili/dynamic.ts @@ -117,7 +117,7 @@ async function handler(ctx) { const response = await got({ method: 'get', - url: `https://api.vc.bilibili.com/dynamic_svr/v1/dynamic_svr/space_history?host_uid=${uid}`, + url: `https://api.vc.bilibili.com/dynamic_svr/v2/dynamic_svr/space_history?host_uid=${uid}`, headers: { Referer: `https://space.bilibili.com/${uid}/`, }, From 6e75de7946da02608aa7984a405c8413632e1ed4 Mon Sep 17 00:00:00 2001 From: lyqluis <39592732+lyqluis@users.noreply.github.com> Date: Mon, 25 Mar 2024 22:57:46 +0800 Subject: [PATCH 5/7] fix(route): /163/dy2/:id pic shows correctly (#14955) --- lib/routes/163/dy2.ts | 7 ++++--- lib/routes/163/utils.ts | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/routes/163/dy2.ts b/lib/routes/163/dy2.ts index 50d7ebf90..527e94083 100644 --- a/lib/routes/163/dy2.ts +++ b/lib/routes/163/dy2.ts @@ -11,7 +11,7 @@ export const route: Route = { path: '/dy2/:id', categories: ['new-media'], example: '/163/dy2/T1555591616739', - parameters: { id: 'id，该网易号主页网址最后一项html的文件名' }, + parameters: { id: 'id，该网易号主页网址最后一项 html 的文件名' }, features: { requireConfig: false, requirePuppeteer: false, @@ -21,7 +21,7 @@ export const route: Route = { supportScihub: false, }, name: '网易号（通用）', - maintainers: ['mjysci'], + maintainers: ['mjysci', 'lyqluis'], handler, description: `优先使用方法一，若是网易号搜索页面搜不到的小众网易号（文章页面不含\`data-wemediaid\`）则可使用此法。 触发反爬会只抓取到标题，建议自建。`, @@ -43,11 +43,12 @@ async function handler(ctx) { .toArray() .map((item) => { item = $(item); + const itemImg = item.find('a.img img'); return { title: item.find('h4 a').text(), link: item.find('a').first().attr('href'), pubDate: timezone(parseDate(item.find('.time').text()), 8), - imgsrc: item.find('a img').attr('src'), + imgsrc: itemImg.attr('src') ?? itemImg.attr('_src'), }; }); diff --git a/lib/routes/163/utils.ts b/lib/routes/163/utils.ts index 61fa5beae..b9f9bc7d8 100644 --- a/lib/routes/163/utils.ts +++ b/lib/routes/163/utils.ts @@ -26,8 +26,9 @@ const parseDyArticle = (charset, item, tryGet) => } }); + const imgUrl = new URL(item.imgsrc); item.description = art(path.join(__dirname, 'templates/dy.art'), { - imgsrc: item.imgsrc?.split('?')[0], + imgsrc: imgUrl.searchParams.get('url'), postBody: $('.post_body').html(), }); From 0ffd4a241dd15925f5cad5d699d16255f4085ec4 Mon Sep 17 00:00:00 2001 From: Tony Date: Mon, 25 Mar 2024 23:11:08 +0800 Subject: [PATCH 6/7] chore: fix build assets action --- .github/workflows/build-assets.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index 7ef73eafe..4f2c38621 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -34,8 +34,6 @@ jobs: run: pnpm i - name: Build assets run: pnpm build - - name: Build docs - run: pnpm build:docs - name: Deploy uses: peaceiris/actions-gh-pages@v3 with: @@ -43,6 +41,10 @@ jobs: publish_dir: ./assets user_name: 'github-actions[bot]' user_email: '41898282+github-actions[bot]@users.noreply.github.com' + # prevent deleting build/test-full-routes.json which will break build:docs + keep_files: true + - name: Build docs + run: pnpm build:docs - name: Checkout docs uses: actions/checkout@v4 with: From 95a5e5025464fa7a523231b5edfbd696cf7ee95f Mon Sep 17 00:00:00 2001 From: LMark <40017222+ladeng07@users.noreply.github.com> Date: Mon, 25 Mar 2024 23:55:33 +0800 Subject: [PATCH 7/7] =?UTF-8?q?feat(route):=20Add=20=E5=8C=97=E4=BA=AC?= =?UTF-8?q?=E5=B8=88=E8=8C=83=E5=A4=A7=E5=AD=A6=E6=95=99=E5=8A=A1=E9=83=A8?= =?UTF-8?q?=20(#14959)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/routes/bnu/jwb.ts | 56 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/routes/bnu/jwb.ts diff --git a/lib/routes/bnu/jwb.ts b/lib/routes/bnu/jwb.ts new file mode 100644 index 000000000..5d190eeac --- /dev/null +++ b/lib/routes/bnu/jwb.ts @@ -0,0 +1,56 @@ +import got from '@/utils/got'; +import { load } from 'cheerio'; +import { parseDate } from '@/utils/parse-date'; +import cache from '@/utils/cache'; + +export const route: Route = { + path: '/jwb', + categories: ['university'], + example: '/bnu/jwb', + parameters: {}, + radar: [ + { + source: ['jwb.bnu.edu.cn'], + }, + ], + name: '教务部（研究生院）', + maintainers: ['ladeng07'], + handler, + url: 'jwb.bnu.edu.cn/tzgg/index.htm', +}; + +async function handler() { + const link = 'https://jwb.bnu.edu.cn/tzgg/index.htm'; + const response = await got(link); + const $ = load(response.data); + const list = $('.article-list .boxlist ul li') + .toArray() + .map((e) => { + e = $(e); + const a = e.find('a'); + return { + title: e.find('a span').text(), + link: a.attr('href').startsWith('http') ? a.attr('href') : 'https://jwb.bnu.edu.cn' + a.attr('href').substring(2), + pubDate: parseDate(e.find('span.fr.text-muted').text(), 'YYYY-MM-DD'), + }; + }); + + const out = await Promise.all( + list.map((item) => + cache.tryGet(item.link, async () => { + const response = await got(item.link); + const $ = load(response.data); + item.author = '北京师范大学教务部'; + item.description = $('.contenttxt').html(); + return item; + }) + ) + ); + + return { + title: '北京师范大学教务部', + link, + description: '北京师范大学教务部最新通知', + item: out, + }; +}