Files
RSSHub/lib/v2/bjfu/it/utils.js
T
Tony 6d901cb7dc style(eslint): add eslint-unicorn (#14257)
* style: add eslint-unicorn

* style: fix unicorn/no-useless-spread

* style: fix unicorn/no-useless-promise-resolve-reject

* style: fix unicorn/no-for-loop

* fix: codeql bad HTML filtering regexp

* fix: codeql incomplete replace

* fix: unicorn/no-abusive-eslint-disable

* style: fix unicorn/no-new-array

* style: fix unicorn/no-typeof-undefined

* style: fix unicorn/no-zero-fractions

* style: fix unicorn/no-empty-file

* style: fix unicorn/prefer-date-now

* revert: auto fix unicorn/prefer-switch on lib/v2/kuaidi100/utils.js

* style: fix unicorn/prefer-array-find

* style: fix unicorn/prefer-array-flat

* style: fix unicorn/prefer-array-flat-map

* style: fix unicorn/prefer-at

* style: fix unicorn/prefer-string-starts-ends-with

* style: fix unicorn/prefer-includes

* fix: codeql URL substring sanitization

* style: fix unicorn/prefer-optional-catch-binding

* style: fix unicorn/catch-error-name

* style: fix unicorn/escape-case

* style: fix unicorn/prefer-native-coercion-functions

* style: fix unicorn/prefer-regexp-test

* style: fix unicorn/require-array-join-separator

* style: fix unicorn/prefer-math-trunc

* style: fix unicorn/prefer-negative-index

* style: fix unicorn/prefer-dom-node-dataset

* style: fix unicorn/prefer-dom-node-text-content

* style: fix unicorn/prefer-query-selector

* style: fix unicorn/no-array-for-each

* style: fix unicorn/no-negated-condition

* style: fix unicorn/prefer-add-event-listener

* style: fix unicorn/import-style

* style: fix prefer-regex-literals

* style: disable unicorn/no-useless-switch-case

* style: disable unicorn/text-encoding-identifier-case

* style: fix unicorn/prefer-set-has

* style: fix unicorn/prefer-spread

* revert: auto fix on lib/routes/universities/ynnu/edu/base64.js

* style: fix unicorn/no-useless-undefined

* style: fix unicorn/no-array-push-push

* style: fix unicorn/no-useless-undefined again

* style: fix unicorn/no-lonely-if

* style: fix unicorn/prefer-reflect-apply

* style: fix unicorn/switch-case-braces

* style: fix unicorn/prefer-switch

* style: fix unicorn/prefer-array-some

* fix: deepscan UNUSED_VAR_ASSIGN

* style: fix unicorn/prefer-ternary

* fix: follow-up of unicorn/prefer-ternary

* revert: auto fix of unicorn/prefer-string-slice for substring()

* style: disable unicorn/prefer-string-slice

fix: auto fix slice over deprecated substr

* style: fix unicorn/throw-new-error

* style: fix unicorn/filename-case

* test: fix dateParser renaming

* style: fix unicorn/better-regex

* style: fix unicorn/prefer-string-replace-all

* fix(deps): add sanitize-html

* style: fix no-prototype-builtins

* style: fix unicorn/consistent-destructuring

* style: fix unicorn/consistent-function-scoping

* style: fix unicorn/prefer-regexp-test

* style: fix unicorn/prefer-logical-operator-over-ternary

* style: fix unicorn/no-array-callback-reference

* style: add prefer-object-has-own

* style: warn unicorn/no-empty-file

* style: fix unicorn/prefer-number-properties

* style: fix no-useless-undefined again

* style: fix unicorn/numeric-separators-style

* style: disable unicorn/no-array-callback-reference

false postive with cheerio
2024-01-18 20:43:40 +08:00

74 lines
2.2 KiB
JavaScript

const got = require('@/utils/got');
const cheerio = require('cheerio');
const iconv = require('iconv-lite');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
// 完整文章页
async function load(link) {
let response;
try {
response = await got.get(link, {
responseType: 'buffer',
});
} catch {
return { description: '' };
}
const data = response.data; // 不用转码
// 加载文章内容
let $ = cheerio.load(iconv.decode(data, 'utf-8'));
const charset = $('meta[charset]').attr('charset');
if (charset?.toLowerCase() !== 'utf-8') {
$ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
}
// 提取内容
const description = ($('.template-body').length ? $('.template-body').html() : '') + ($('.template-tail').length ? $('.template-tail').html() : '');
// 返回解析的结果
return { description };
}
const ProcessFeed = (base, list, caches) =>
// 使用 Promise.all() 进行 async 并发
Promise.all(
// 遍历每一篇文章
list.map((item) => {
const $ = cheerio.load(item);
const $title = $('a');
// 还原相对链接为绝对链接
const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
// 解析日期
const pubDate = timezone(
parseDate(
$('span')
.text()
.match(/\d{4}-\d{2}-\d{2}/)
),
+8
);
// 使用tryGet方法从缓存获取内容。
// 当缓存中无法获取到链接内容的时候,则使用load方法加载文章内容。
return caches.tryGet(itemUrl, async () => {
const { description } = await load(itemUrl);
// 列表上提取到的信息
return {
title: $title.text(),
link: itemUrl,
author: '北林信息',
description,
pubDate,
};
});
})
);
module.exports = {
ProcessFeed,
};