fix(route): dcard (#10739)
* fix: dcard use puppeteer * docs: add pptr attr * fix: use cookies * docs: add warning
This commit is contained in:
Tony
GitHub
parent
655b899715
commit
ba2df5d3d9
10
docs/bbs.md
10
docs/bbs.md
|
|
@ -130,13 +130,19 @@ pageClass: routes
|
|||
|
||||
## Dcard
|
||||
|
||||
::: warning 注意
|
||||
|
||||
僅能透過台灣 IP 抓取。
|
||||
|
||||
:::
|
||||
|
||||
### 首頁帖子
|
||||
|
||||
<Route author="DIYgod" example="/dcard/posts/popular" path="/dcard/posts/:type?" :paramsDesc="['排序，popular 熱門；latest 最新，默認為 latest']" radar="1" rssbud="1" anticrawler="1"/>
|
||||
<Route author="DIYgod" example="/dcard/posts/popular" path="/dcard/posts/:type?" :paramsDesc="['排序，popular 熱門；latest 最新，默認為 latest']" radar="1" rssbud="1" anticrawler="1" puppeteer="1"/>
|
||||
|
||||
### 板塊帖子
|
||||
|
||||
<Route author="HenryQW" example="/dcard/funny/popular" path="/dcard/:section/:type?" :paramsDesc="['板塊名稱，URL 中獲得', '排序，popular 熱門；latest 最新，默認為 latest']" radar="1" rssbud="1" anticrawler="1"/>
|
||||
<Route author="HenryQW" example="/dcard/funny/popular" path="/dcard/:section/:type?" :paramsDesc="['板塊名稱，URL 中獲得', '排序，popular 熱門；latest 最新，默認為 latest']" radar="1" rssbud="1" anticrawler="1" puppeteer="1"/>
|
||||
|
||||
## Discuz
|
||||
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
const got = require('@/utils/got');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
const utils = require('./utils');
|
||||
|
||||
module.exports = async (ctx) => {
|
||||
const { type, section = 'posts' } = ctx.params;
|
||||
const { type = 'latest', section = 'posts' } = ctx.params;
|
||||
const limit = ctx.query.limit ? Number(ctx.query.limit) : 30;
|
||||
const browser = await require('@/utils/puppeteer')();
|
||||
|
||||
let link = `https://www.dcard.tw/f`;
|
||||
|
||||
let api = `https://www.dcard.tw/service/api/v2`;
|
||||
|
||||
let title = `Dcard - `;
|
||||
|
||||
if (section !== 'posts' && section !== 'popular' && section !== 'latest') {
|
||||
|
|
@ -15,31 +15,52 @@ module.exports = async (ctx) => {
|
|||
api += `/forums/${section}`;
|
||||
title += `${section} - `;
|
||||
}
|
||||
|
||||
api += `/posts`;
|
||||
|
||||
if (type === 'popular') {
|
||||
link += '?latest=false';
|
||||
api += '?popular=true';
|
||||
title += '熱門';
|
||||
} else if (type === 'latest' || !type) {
|
||||
} else {
|
||||
link += '?latest=true';
|
||||
api += '?popular=false';
|
||||
title += '最新';
|
||||
}
|
||||
|
||||
const response = await got(`${api}&limit=30`, {
|
||||
headers: {
|
||||
'user-agent': 'dcard-web',
|
||||
},
|
||||
const page = await browser.newPage();
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
|
||||
});
|
||||
await page.setExtraHTTPHeaders({
|
||||
referer: `https://www.dcard.tw/f/${section}`,
|
||||
});
|
||||
|
||||
const items = await utils.ProcessFeed(response.data, ctx.cache);
|
||||
await page.goto(`${api}&limit=100`);
|
||||
await page.waitForSelector('body > pre');
|
||||
const response = await page.evaluate(() => document.querySelector('body > pre').innerText);
|
||||
const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false);
|
||||
await page.close();
|
||||
|
||||
const data = JSON.parse(response);
|
||||
const items = data.map((item) => ({
|
||||
title: `「${item.forumName}」${item.title}`,
|
||||
link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
|
||||
description: item.excerpt,
|
||||
author: `${item.school || '匿名'}．${item.gender === 'M' ? '男' : '女'}`,
|
||||
pubDate: parseDate(item.createdAt),
|
||||
category: [item.forumName, ...item.topics],
|
||||
forumAlias: item.forumAlias,
|
||||
id: item.id,
|
||||
}));
|
||||
|
||||
// parse fulltext for first `limit` items
|
||||
const result = await utils.ProcessFeed(items, cookies, browser, limit, ctx.cache);
|
||||
await browser.close();
|
||||
|
||||
ctx.state.data = {
|
||||
title,
|
||||
link,
|
||||
description: '不想錯過任何有趣的話題嗎？趕快加入我們吧！',
|
||||
item: items,
|
||||
item: result,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,51 +1,48 @@
|
|||
const got = require('@/utils/got');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
const asyncPool = require('tiny-async-pool');
|
||||
|
||||
const ProcessFeed = async (list, cache) => {
|
||||
const result = await Promise.all(
|
||||
list.map(async (item) => {
|
||||
const url = `https://www.dcard.tw/service/api/v2/posts/${item.id}`;
|
||||
const ProcessFeed = async (items, cookies, browser, limit, cache) => {
|
||||
let newCookies = [];
|
||||
const result = [];
|
||||
for await (const item of asyncPool(3, items.slice(0, limit), async (i) => {
|
||||
const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`;
|
||||
const content = await cache.tryGet(`dcard:${i.id}`, async () => {
|
||||
let response;
|
||||
// try catch 处理被删除的帖子
|
||||
try {
|
||||
const page = await browser.newPage();
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
|
||||
});
|
||||
await page.setExtraHTTPHeaders({
|
||||
referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
|
||||
});
|
||||
await page.setCookie(...cookies);
|
||||
await page.goto(url);
|
||||
await page.waitForSelector('body > pre');
|
||||
response = await page.evaluate(() => document.querySelector('body > pre').innerText);
|
||||
newCookies = await page.cookies();
|
||||
await page.close();
|
||||
|
||||
const content = await cache.tryGet(`dcard:${item.id}`, async () => {
|
||||
let response;
|
||||
const data = JSON.parse(response);
|
||||
let body = data.content;
|
||||
body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `<img src="${m}">`);
|
||||
body = body.replace(/(?=https?:\/\/).*(?<!jpe?g"?>?)$/gim, (m) => `<a href="${m}">${m}</a>`);
|
||||
body = body.replace(/\n/g, '<br>');
|
||||
|
||||
// try catch 处理被删除的帖子
|
||||
return body;
|
||||
} catch (error) {
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
response = await got(url, {
|
||||
headers: {
|
||||
'user-agent': 'dcard-web',
|
||||
},
|
||||
});
|
||||
|
||||
let body = response.data.content;
|
||||
|
||||
body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `<img src="${m}">`);
|
||||
|
||||
body = body.replace(/(?=https?:\/\/).*(?<!jpe?g"?>?)$/gim, (m) => `<a href="${m}">${m}</a>`);
|
||||
|
||||
body = body.replace(/\n/g, '<br>');
|
||||
|
||||
return body;
|
||||
} catch (error) {
|
||||
return '';
|
||||
}
|
||||
});
|
||||
|
||||
const single = {
|
||||
title: `「${item.forumName}」${item.title}`,
|
||||
link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
|
||||
description: content,
|
||||
author: item.school || '匿名',
|
||||
pubDate: parseDate(item.createdAt),
|
||||
category: [item.forumName, ...item.topics],
|
||||
};
|
||||
|
||||
return single;
|
||||
})
|
||||
);
|
||||
|
||||
return result.filter((f) => f.description);
|
||||
i.description = content;
|
||||
return i;
|
||||
})) {
|
||||
result.push(item);
|
||||
}
|
||||
await cache.set('dcard:cookies', newCookies, 3600);
|
||||
return [...result, ...items.slice(limit)];
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
|
|
|
|||
Loading…
Reference in New Issue