From 5e63059947b44f98797a67f44fda9fefe720149a Mon Sep 17 00:00:00 2001 From: Andvari <31068367+dzx-dzx@users.noreply.github.com> Date: Fri, 14 Jul 2023 23:16:25 +0800 Subject: [PATCH] =?UTF-8?q?fix(route):=20=E6=9B=B4=E6=96=B0=E8=81=94?= =?UTF-8?q?=E5=90=88=E6=97=A9=E6=8A=A5=E6=97=B6=E9=97=B4=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E6=96=B9=E6=B3=95=20(#12789)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update util.js * Update util.js * Update util.js * Update util.js * Update util.js * refactor: use cache.tryGet * Update util.js * Update util.js * Update util.js * fix: zaobao content order fix #10309 --------- Co-authored-by: TonyRL --- lib/v2/zaobao/util.js | 69 ++++++++++++++++++++++++------------------- package.json | 1 + pnpm-lock.yaml | 7 +++++ 3 files changed, 46 insertions(+), 31 deletions(-) diff --git a/lib/v2/zaobao/util.js b/lib/v2/zaobao/util.js index 0e4da9e9e..0d990730c 100644 --- a/lib/v2/zaobao/util.js +++ b/lib/v2/zaobao/util.js @@ -4,6 +4,7 @@ const date = require('@/utils/date'); const timezone = require('@/utils/timezone'); const { art } = require('@/utils/render'); const path = require('path'); +const { base32 } = require('rfc4648'); const baseUrl = 'https://www.zaobao.com'; const got_ins = got.extend({ @@ -35,22 +36,14 @@ const parseList = async (ctx, sectionUrl) => { data = $('.clearfix').find('.list-block'); } - const title = $('#breadcrumbs > a') - .toArray() - .reduce((acc, cV, cI) => { - if (cI > 0) { - return acc + '-' + $(cV).text(); - } - return ''; - }, ''); + const title = $('meta[property="og:title"]').attr('content'); + const resultList = await Promise.all( - data.toArray().map(async (item) => { + data.toArray().map((item) => { const $item = $(item); // addBack: for HK version let link = $item.find('a').addBack('a')[0].attribs.href; - let resultItem = {}; - if (link[0] !== '/') { // https://www.zaobao.com/interactive-graphics const title = $item.find('a').text(); @@ -71,31 +64,29 @@ const parseList = async (ctx, sectionUrl) => { pubDate = pubDate === dateString ? undefined : timezone(pubDate, +8); } - resultItem = { + return { title, description: title, pubDate, link, }; - return Promise.resolve(resultItem); - } else { - link = baseUrl + link; } - const value = await ctx.cache.get(link); + link = baseUrl + link; - if (value) { - resultItem = JSON.parse(value); - } else { + return ctx.cache.tryGet(link, async () => { const article = await got_ins.get(link); const $1 = cheerio.load(article.data); - let time = $1('.date-published', '.content').slice(0, 1).text().trim(); - if (!time) { - // for HK version - time = $1('.author-info').find('.mgt10').slice(0, 1).text().trim(); - } - time = time.replace('年', '-').replace('月', '-').replace('日', '').replace('发布 /', ''); + const time = (() => { + if ($1("head script[type='application/json']").text() !== '') { + // SG + return new Date(Number(JSON.parse($1("head script[type='application/json']").text())?.articleDetails?.created) * 1000); + } else { + // HK + return new Date(JSON.parse($1("head script[type='application/ld+json']").eq(1).text())?.datePublished); + } + })(); $1('.overlay-microtransaction').remove(); $1('#video-freemium-player').remove(); @@ -104,6 +95,7 @@ const parseList = async (ctx, sectionUrl) => { let articleBodyNode = $1('.article-content-rawhtml'); if (articleBodyNode.length === 0) { // for HK version + orderContent($1('.article-body')); articleBodyNode = $1('.article-body'); } @@ -152,20 +144,17 @@ const parseList = async (ctx, sectionUrl) => { }); } - resultItem = { + return { // <- for SG version -> for HK version title: $1('h1', '.content').text().trim() || $1('h1.article-title').text(), description: art(path.join(__dirname, 'templates/zaobao.art'), { articleBody, imageDataArray, }), - pubDate: timezone(time, +8), // zaobao seems always UTC+8 + pubDate: time, link, }; - ctx.cache.set(link, JSON.stringify(resultItem)); - } - - return Promise.resolve(resultItem); + }); }) ); return { @@ -173,6 +162,24 @@ const parseList = async (ctx, sectionUrl) => { resultList, }; }; + +const orderContent = (parent) => { + parent + .children() + .toArray() + .sort((a, b) => { + const index = Buffer.from(base32.parse('GM======')).toString(); // substring(3) + a = Buffer.from(base32.parse(parent.find(a).data('s').substring(index))).toString(); + b = Buffer.from(base32.parse(parent.find(b).data('s').substring(index))).toString(); + return a - b; + }) + .forEach((e, i) => { + parent.find(e).attr('s', i); + parent.append(e); + }); +}; + module.exports = { parseList, + orderContent, }; diff --git a/package.json b/package.json index 55e8d6f44..c6b429e37 100644 --- a/package.json +++ b/package.json @@ -128,6 +128,7 @@ "rand-user-agent": "1.0.109", "re2": "1.19.1", "require-all": "3.0.0", + "rfc4648": "1.5.2", "rss-parser": "3.13.0", "showdown": "2.1.0", "simplecc-wasm": "0.1.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 06a74bb87..d4c467eaf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -160,6 +160,9 @@ dependencies: require-all: specifier: 3.0.0 version: 3.0.0 + rfc4648: + specifier: 1.5.2 + version: 1.5.2 rss-parser: specifier: 3.13.0 version: 3.13.0 @@ -6795,6 +6798,10 @@ packages: engines: {iojs: '>=1.0.0', node: '>=0.10.0'} dev: true + /rfc4648@1.5.2: + resolution: {integrity: sha512-tLOizhR6YGovrEBLatX1sdcuhoSCXddw3mqNVAcKxGJ+J0hFeJ+SjeWCv5UPA/WU3YzWPPuCVYgXBKZUPGpKtg==} + dev: false + /rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} hasBin: true