fix: /flyert/creditcard 493错误 (#18974)
* 修正域名,增加随机时间控制,限制并发为2 * 使用new URL替换url.resolve并增加p-limit库 * 使用官方推荐的库实现相同的功能 * 使用官方已有函数减少重复代码 * chore: remove p-limit --------- Co-authored-by: 独自等待 <liliang@LL-147.local>
This commit is contained in:
parent
5a2e38580c
commit
56436caef0
|
|
@ -6,7 +6,7 @@ import util from './utils';
|
|||
import iconv from 'iconv-lite';
|
||||
|
||||
const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
|
||||
const host = 'https://www.flyert.com';
|
||||
const host = 'https://www.flyert.com.cn';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/creditcard/:bank',
|
||||
|
|
@ -23,7 +23,7 @@ export const route: Route = {
|
|||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['flyert.com/'],
|
||||
source: ['flyert.com.cn/'],
|
||||
},
|
||||
],
|
||||
name: '信用卡',
|
||||
|
|
@ -129,7 +129,7 @@ async function handler(ctx) {
|
|||
|
||||
return {
|
||||
title: `飞客茶馆信用卡 - ${bankname}`,
|
||||
link: 'https://www.flyert.com/',
|
||||
link: 'https://www.flyert.com.cn/',
|
||||
description: `飞客茶馆信用卡 - ${bankname}`,
|
||||
item: result,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import type { Namespace } from '@/types';
|
|||
|
||||
export const namespace: Namespace = {
|
||||
name: '飞客茶馆',
|
||||
url: 'flyert.com',
|
||||
url: 'flyert.com.cn',
|
||||
description: '',
|
||||
lang: 'zh-CN',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,18 +1,22 @@
|
|||
import got from '@/utils/got';
|
||||
import { load } from 'cheerio';
|
||||
import * as url from 'node:url';
|
||||
import iconv from 'iconv-lite';
|
||||
import pMap from 'p-map';
|
||||
import wait from '@/utils/wait';
|
||||
|
||||
const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
|
||||
|
||||
// 加载文章页
|
||||
async function loadContent(link) {
|
||||
// 添加随机延迟,假设延迟时间在1000毫秒到3000毫秒之间
|
||||
const randomDelay = Math.floor(Math.random() * (3000 - 1000 + 1)) + 1000;
|
||||
await wait(randomDelay);
|
||||
|
||||
const response = await got.get(link, {
|
||||
responseType: 'buffer',
|
||||
});
|
||||
const $ = load(gbk2utf8(response.data));
|
||||
|
||||
// 去除全文末尾多与内容
|
||||
// 去除全文末尾多余内容
|
||||
$('.lookMore').remove();
|
||||
$('script, style').remove();
|
||||
$('#loginDialog').remove();
|
||||
|
|
@ -23,7 +27,7 @@ async function loadContent(link) {
|
|||
// 修改图片中的链接
|
||||
firstpost.find('ignore_js_op img').each(function () {
|
||||
$(this).attr('src', $(this).attr('file'));
|
||||
// remove useless atrributes
|
||||
// 移除无用属性
|
||||
for (const attr of ['id', 'aid', 'zoomfile', 'file', 'zoomfile', 'class', 'onclick', 'title', 'inpost', 'alt', 'onmouseover']) {
|
||||
$(this).removeAttr(attr);
|
||||
}
|
||||
|
|
@ -34,23 +38,24 @@ async function loadContent(link) {
|
|||
firstpost.find('ignore_js_op').remove();
|
||||
firstpost.append(images);
|
||||
|
||||
// // 提取内容
|
||||
// 提取内容
|
||||
const description = firstpost.html();
|
||||
|
||||
return { description };
|
||||
}
|
||||
|
||||
const ProcessFeed = (list, caches) => {
|
||||
const host = 'https://www.flyert.com';
|
||||
const host = 'https://www.flyert.com.cn';
|
||||
|
||||
return Promise.all(
|
||||
list.map(async (item) => {
|
||||
return pMap(
|
||||
list,
|
||||
async (item) => {
|
||||
const $ = load(item);
|
||||
|
||||
const $label = $(".comiis_common a[data-track='版块页主题分类']");
|
||||
const $title = $(".comiis_common a[data-track='版块页文章']");
|
||||
// 还原相对链接为绝对链接
|
||||
const itemUrl = url.resolve(host, $title.attr('href'));
|
||||
const itemUrl = new URL($title.attr('href'), host).href;
|
||||
|
||||
// 列表上提取到的信息
|
||||
const single = {
|
||||
|
|
@ -65,8 +70,9 @@ const ProcessFeed = (list, caches) => {
|
|||
|
||||
// 合并解析后的结果集作为该篇文章最终的输出结果
|
||||
return Object.assign({}, single, other);
|
||||
})
|
||||
);
|
||||
},
|
||||
{ concurrency: 2 }
|
||||
); // 设置并发请求数量为 2
|
||||
};
|
||||
|
||||
export default { ProcessFeed };
|
||||
|
|
|
|||
Loading…
Reference in New Issue