* style: add eslint-unicorn * style: fix unicorn/no-useless-spread * style: fix unicorn/no-useless-promise-resolve-reject * style: fix unicorn/no-for-loop * fix: codeql bad HTML filtering regexp * fix: codeql incomplete replace * fix: unicorn/no-abusive-eslint-disable * style: fix unicorn/no-new-array * style: fix unicorn/no-typeof-undefined * style: fix unicorn/no-zero-fractions * style: fix unicorn/no-empty-file * style: fix unicorn/prefer-date-now * revert: auto fix unicorn/prefer-switch on lib/v2/kuaidi100/utils.js * style: fix unicorn/prefer-array-find * style: fix unicorn/prefer-array-flat * style: fix unicorn/prefer-array-flat-map * style: fix unicorn/prefer-at * style: fix unicorn/prefer-string-starts-ends-with * style: fix unicorn/prefer-includes * fix: codeql URL substring sanitization * style: fix unicorn/prefer-optional-catch-binding * style: fix unicorn/catch-error-name * style: fix unicorn/escape-case * style: fix unicorn/prefer-native-coercion-functions * style: fix unicorn/prefer-regexp-test * style: fix unicorn/require-array-join-separator * style: fix unicorn/prefer-math-trunc * style: fix unicorn/prefer-negative-index * style: fix unicorn/prefer-dom-node-dataset * style: fix unicorn/prefer-dom-node-text-content * style: fix unicorn/prefer-query-selector * style: fix unicorn/no-array-for-each * style: fix unicorn/no-negated-condition * style: fix unicorn/prefer-add-event-listener * style: fix unicorn/import-style * style: fix prefer-regex-literals * style: disable unicorn/no-useless-switch-case * style: disable unicorn/text-encoding-identifier-case * style: fix unicorn/prefer-set-has * style: fix unicorn/prefer-spread * revert: auto fix on lib/routes/universities/ynnu/edu/base64.js * style: fix unicorn/no-useless-undefined * style: fix unicorn/no-array-push-push * style: fix unicorn/no-useless-undefined again * style: fix unicorn/no-lonely-if * style: fix unicorn/prefer-reflect-apply * style: fix unicorn/switch-case-braces * style: fix unicorn/prefer-switch * style: fix unicorn/prefer-array-some * fix: deepscan UNUSED_VAR_ASSIGN * style: fix unicorn/prefer-ternary * fix: follow-up of unicorn/prefer-ternary * revert: auto fix of unicorn/prefer-string-slice for substring() * style: disable unicorn/prefer-string-slice fix: auto fix slice over deprecated substr * style: fix unicorn/throw-new-error * style: fix unicorn/filename-case * test: fix dateParser renaming * style: fix unicorn/better-regex * style: fix unicorn/prefer-string-replace-all * fix(deps): add sanitize-html * style: fix no-prototype-builtins * style: fix unicorn/consistent-destructuring * style: fix unicorn/consistent-function-scoping * style: fix unicorn/prefer-regexp-test * style: fix unicorn/prefer-logical-operator-over-ternary * style: fix unicorn/no-array-callback-reference * style: add prefer-object-has-own * style: warn unicorn/no-empty-file * style: fix unicorn/prefer-number-properties * style: fix no-useless-undefined again * style: fix unicorn/numeric-separators-style * style: disable unicorn/no-array-callback-reference false postive with cheerio
75 lines
2.3 KiB
JavaScript
75 lines
2.3 KiB
JavaScript
const got = require('@/utils/got');
|
|
const cheerio = require('cheerio');
|
|
const url = require('url');
|
|
const iconv = require('iconv-lite');
|
|
|
|
const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
|
|
|
|
// 加载文章页
|
|
async function load(link) {
|
|
const response = await got.get(link, {
|
|
responseType: 'buffer',
|
|
});
|
|
const $ = cheerio.load(gbk2utf8(response.data));
|
|
|
|
// 去除全文末尾多与内容
|
|
$('.lookMore').remove();
|
|
$('script, style').remove();
|
|
$('#loginDialog').remove();
|
|
|
|
// 获取第一个帖子对象
|
|
const firstpost = $('.firstpost');
|
|
|
|
// 修改图片中的链接
|
|
firstpost.find('ignore_js_op img').each(function () {
|
|
$(this).attr('src', $(this).attr('file'));
|
|
// remove useless atrributes
|
|
for (const attr of ['id', 'aid', 'zoomfile', 'file', 'zoomfile', 'class', 'onclick', 'title', 'inpost', 'alt', 'onmouseover']) {
|
|
$(this).removeAttr(attr);
|
|
}
|
|
});
|
|
|
|
// 去除全文中图片的多余标签
|
|
const images = firstpost.find('ignore_js_op img');
|
|
firstpost.find('ignore_js_op').remove();
|
|
firstpost.append(images);
|
|
|
|
// // 提取内容
|
|
const description = firstpost.html();
|
|
|
|
return { description };
|
|
}
|
|
|
|
const ProcessFeed = (list, caches) => {
|
|
const host = 'https://www.flyert.com';
|
|
|
|
return Promise.all(
|
|
list.map(async (item) => {
|
|
const $ = cheerio.load(item);
|
|
|
|
const $label = $(".comiis_common a[data-track='版块页主题分类']");
|
|
const $title = $(".comiis_common a[data-track='版块页文章']");
|
|
// 还原相对链接为绝对链接
|
|
const itemUrl = url.resolve(host, $title.attr('href'));
|
|
|
|
// 列表上提取到的信息
|
|
const single = {
|
|
title: $label.text() + '-' + $title.text(),
|
|
link: itemUrl,
|
|
guid: itemUrl,
|
|
};
|
|
|
|
// 使用tryGet方法从缓存获取内容。
|
|
// 当缓存中无法获取到链接内容的时候,则使用load方法加载文章内容。
|
|
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
|
|
|
|
// 合并解析后的结果集作为该篇文章最终的输出结果
|
|
return Object.assign({}, single, other);
|
|
})
|
|
);
|
|
};
|
|
|
|
module.exports = {
|
|
ProcessFeed,
|
|
};
|