* style: add eslint-unicorn * style: fix unicorn/no-useless-spread * style: fix unicorn/no-useless-promise-resolve-reject * style: fix unicorn/no-for-loop * fix: codeql bad HTML filtering regexp * fix: codeql incomplete replace * fix: unicorn/no-abusive-eslint-disable * style: fix unicorn/no-new-array * style: fix unicorn/no-typeof-undefined * style: fix unicorn/no-zero-fractions * style: fix unicorn/no-empty-file * style: fix unicorn/prefer-date-now * revert: auto fix unicorn/prefer-switch on lib/v2/kuaidi100/utils.js * style: fix unicorn/prefer-array-find * style: fix unicorn/prefer-array-flat * style: fix unicorn/prefer-array-flat-map * style: fix unicorn/prefer-at * style: fix unicorn/prefer-string-starts-ends-with * style: fix unicorn/prefer-includes * fix: codeql URL substring sanitization * style: fix unicorn/prefer-optional-catch-binding * style: fix unicorn/catch-error-name * style: fix unicorn/escape-case * style: fix unicorn/prefer-native-coercion-functions * style: fix unicorn/prefer-regexp-test * style: fix unicorn/require-array-join-separator * style: fix unicorn/prefer-math-trunc * style: fix unicorn/prefer-negative-index * style: fix unicorn/prefer-dom-node-dataset * style: fix unicorn/prefer-dom-node-text-content * style: fix unicorn/prefer-query-selector * style: fix unicorn/no-array-for-each * style: fix unicorn/no-negated-condition * style: fix unicorn/prefer-add-event-listener * style: fix unicorn/import-style * style: fix prefer-regex-literals * style: disable unicorn/no-useless-switch-case * style: disable unicorn/text-encoding-identifier-case * style: fix unicorn/prefer-set-has * style: fix unicorn/prefer-spread * revert: auto fix on lib/routes/universities/ynnu/edu/base64.js * style: fix unicorn/no-useless-undefined * style: fix unicorn/no-array-push-push * style: fix unicorn/no-useless-undefined again * style: fix unicorn/no-lonely-if * style: fix unicorn/prefer-reflect-apply * style: fix unicorn/switch-case-braces * style: fix unicorn/prefer-switch * style: fix unicorn/prefer-array-some * fix: deepscan UNUSED_VAR_ASSIGN * style: fix unicorn/prefer-ternary * fix: follow-up of unicorn/prefer-ternary * revert: auto fix of unicorn/prefer-string-slice for substring() * style: disable unicorn/prefer-string-slice fix: auto fix slice over deprecated substr * style: fix unicorn/throw-new-error * style: fix unicorn/filename-case * test: fix dateParser renaming * style: fix unicorn/better-regex * style: fix unicorn/prefer-string-replace-all * fix(deps): add sanitize-html * style: fix no-prototype-builtins * style: fix unicorn/consistent-destructuring * style: fix unicorn/consistent-function-scoping * style: fix unicorn/prefer-regexp-test * style: fix unicorn/prefer-logical-operator-over-ternary * style: fix unicorn/no-array-callback-reference * style: add prefer-object-has-own * style: warn unicorn/no-empty-file * style: fix unicorn/prefer-number-properties * style: fix no-useless-undefined again * style: fix unicorn/numeric-separators-style * style: disable unicorn/no-array-callback-reference false postive with cheerio
74 lines
2.2 KiB
JavaScript
74 lines
2.2 KiB
JavaScript
const got = require('@/utils/got');
|
|
const cheerio = require('cheerio');
|
|
const iconv = require('iconv-lite');
|
|
const { parseDate } = require('@/utils/parse-date');
|
|
const timezone = require('@/utils/timezone');
|
|
|
|
// 完整文章页
|
|
async function load(link) {
|
|
let response;
|
|
try {
|
|
response = await got.get(link, {
|
|
responseType: 'buffer',
|
|
});
|
|
} catch {
|
|
return { description: '' };
|
|
}
|
|
|
|
const data = response.data; // 不用转码
|
|
|
|
// 加载文章内容
|
|
let $ = cheerio.load(iconv.decode(data, 'utf-8'));
|
|
const charset = $('meta[charset]').attr('charset');
|
|
if (charset?.toLowerCase() !== 'utf-8') {
|
|
$ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
|
|
}
|
|
|
|
// 提取内容
|
|
const description = ($('.template-body').length ? $('.template-body').html() : '') + ($('.template-tail').length ? $('.template-tail').html() : '');
|
|
|
|
// 返回解析的结果
|
|
return { description };
|
|
}
|
|
|
|
const ProcessFeed = (base, list, caches) =>
|
|
// 使用 Promise.all() 进行 async 并发
|
|
Promise.all(
|
|
// 遍历每一篇文章
|
|
list.map((item) => {
|
|
const $ = cheerio.load(item);
|
|
|
|
const $title = $('a');
|
|
// 还原相对链接为绝对链接
|
|
const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
|
|
|
|
// 解析日期
|
|
const pubDate = timezone(
|
|
parseDate(
|
|
$('span')
|
|
.text()
|
|
.match(/\d{4}-\d{2}-\d{2}/)
|
|
),
|
|
+8
|
|
);
|
|
|
|
// 使用tryGet方法从缓存获取内容。
|
|
// 当缓存中无法获取到链接内容的时候,则使用load方法加载文章内容。
|
|
return caches.tryGet(itemUrl, async () => {
|
|
const { description } = await load(itemUrl);
|
|
|
|
// 列表上提取到的信息
|
|
return {
|
|
title: $title.text(),
|
|
link: itemUrl,
|
|
author: '北林信息',
|
|
description,
|
|
pubDate,
|
|
};
|
|
});
|
|
})
|
|
);
|
|
module.exports = {
|
|
ProcessFeed,
|
|
};
|