From a47a886f291ae7f526babca7985dc00422914efc Mon Sep 17 00:00:00 2001 From: Felix Hsu Date: Wed, 1 Feb 2023 00:52:03 +0800 Subject: [PATCH] fix(route): linkedin.cn Fix the job desc section formatting error with renderer by attributes. (#11736) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(route): xiaomiyoupin crowdfunding error and migrate to v2 * initial commit * add searchHit * add jobPosting * linkedincn * add linkedincn ok * feat(route): 中国新闻周刊 (#11665) * feat(route): 中国新闻周刊 * fix: decode via iconv instead of puppeteer * fix: cache full text request * Update lib/v2/inewsweek/index.js * fix: exact publish time * fix: timezone * feat(route): 修复 HelloGitHub 的 月刊 路由 (#11675) * feat(route): 添加 百度股市通 * fix: 修改 百度股市通 路径 * docs: 添加 百度股市通 文档 * feat: 添加 radar 支持 * fix: 按字母顺序插入新路由 * feat(route): 新增 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 * fix: 优化 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 的标题 * fix: 修复 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 部分情况下的非空判断 * fix: 修复 地区名称标题的问题 * fix: 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 的 guid 增加 pubDate * fix: 修复 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 guid 中添加 pubDate * fix: 修改 腾讯新闻 - 新型冠状病毒肺炎疫情实时追踪 的 title * feat(route): 修复 HelloGitHub 的 月刊 路由 * fix: remove guid * fix(route): x6d (#11676) * chore(deps): bump rand-user-agent from 1.0.101 to 1.0.102 (#11678) Bumps [rand-user-agent](https://github.com/WebScrapingAPI/rand-user-agent) from 1.0.101 to 1.0.102. - [Release notes](https://github.com/WebScrapingAPI/rand-user-agent/releases) - [Commits](https://github.com/WebScrapingAPI/rand-user-agent/commits) --- updated-dependencies: - dependency-name: rand-user-agent dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump @sentry/node from 7.31.1 to 7.32.1 (#11680) Bumps [@sentry/node](https://github.com/getsentry/sentry-javascript) from 7.31.1 to 7.32.1. - [Release notes](https://github.com/getsentry/sentry-javascript/releases) - [Changelog](https://github.com/getsentry/sentry-javascript/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-javascript/compare/7.31.1...7.32.1) --- updated-dependencies: - dependency-name: "@sentry/node" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump jsdom from 21.0.0 to 21.1.0 (#11681) Bumps [jsdom](https://github.com/jsdom/jsdom) from 21.0.0 to 21.1.0. - [Release notes](https://github.com/jsdom/jsdom/releases) - [Changelog](https://github.com/jsdom/jsdom/blob/master/Changelog.md) - [Commits](https://github.com/jsdom/jsdom/compare/21.0.0...21.1.0) --- updated-dependencies: - dependency-name: jsdom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps): bump puppeteer from 19.5.2 to 19.6.0 (#11679) Bumps [puppeteer](https://github.com/puppeteer/puppeteer) from 19.5.2 to 19.6.0. - [Release notes](https://github.com/puppeteer/puppeteer/releases) - [Changelog](https://github.com/puppeteer/puppeteer/blob/main/release-please-config.json) - [Commits](https://github.com/puppeteer/puppeteer/compare/puppeteer-v19.5.2...puppeteer-v19.6.0) --- updated-dependencies: - dependency-name: puppeteer dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * feat(route): famitsu (#11682) * fix(route): xiaomiyoupin crowdfunding error and migrate to v2 (#11677) * fix(route): xiaomiyoupin crowdfunding error and migrate to v2 * Update lib/v2/xiaomiyoupin/maintainer.js * Update docs/shopping.md * use single api endpoint for both module * add pubDate() * fix(route): ikea (#11685) * chore(deps): bump @sentry/node from 7.32.1 to 7.33.0 (#11687) Bumps [@sentry/node](https://github.com/getsentry/sentry-javascript) from 7.32.1 to 7.33.0. - [Release notes](https://github.com/getsentry/sentry-javascript/releases) - [Changelog](https://github.com/getsentry/sentry-javascript/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-javascript/compare/7.32.1...7.33.0) --- updated-dependencies: - dependency-name: "@sentry/node" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump jest from 29.3.1 to 29.4.0 (#11686) Bumps [jest](https://github.com/facebook/jest/tree/HEAD/packages/jest) from 29.3.1 to 29.4.0. - [Release notes](https://github.com/facebook/jest/releases) - [Changelog](https://github.com/facebook/jest/blob/main/CHANGELOG.md) - [Commits](https://github.com/facebook/jest/commits/v29.4.0/packages/jest) --- updated-dependencies: - dependency-name: jest dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(route): bloomberg videopage parse error (#11688) * fix(route): clickme (#11689) * fix(route): lofter (#11696) * fix(route): caixin blog (#11695) * fix(route): caixin blog * fix: url * chore(deps): bump ioredis from 5.2.5 to 5.3.0 (#11697) Bumps [ioredis](https://github.com/luin/ioredis) from 5.2.5 to 5.3.0. - [Release notes](https://github.com/luin/ioredis/releases) - [Changelog](https://github.com/luin/ioredis/blob/main/CHANGELOG.md) - [Commits](https://github.com/luin/ioredis/compare/v5.2.5...v5.3.0) --- updated-dependencies: - dependency-name: ioredis dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(route): bioone (#11699) * feat(route): github wiki history (#11700) * feat(route): wuhan (#11701) Co-authored-by: nczitzk * chore(deps): bump puppeteer from 19.6.0 to 19.6.1 (#11702) Bumps [puppeteer](https://github.com/puppeteer/puppeteer) from 19.6.0 to 19.6.1. - [Release notes](https://github.com/puppeteer/puppeteer/releases) - [Changelog](https://github.com/puppeteer/puppeteer/blob/main/release-please-config.json) - [Commits](https://github.com/puppeteer/puppeteer/compare/puppeteer-v19.6.0...puppeteer-v19.6.1) --- updated-dependencies: - dependency-name: puppeteer dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * chore(deps-dev): bump jest from 29.4.0 to 29.4.1 (#11703) Bumps [jest](https://github.com/facebook/jest/tree/HEAD/packages/jest) from 29.4.0 to 29.4.1. - [Release notes](https://github.com/facebook/jest/releases) - [Changelog](https://github.com/facebook/jest/blob/main/CHANGELOG.md) - [Commits](https://github.com/facebook/jest/commits/v29.4.1/packages/jest) --- updated-dependencies: - dependency-name: jest dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(route): fix date err and refactor to v2 (#11705) * fix(route): fix date err and refactor to v2 * docs: add en docs * fix: typo * feat(route): add mixcloud (#11672) * feat(route): add mixcloud * docs * feat(route): add mixcloud * convert linebreaks to
* docs: add supportPodcast * codestyle: use template literals * refactor: split graph ql queries * fix(route): 软餐 (#11708) * feat(route): gelonghui (#11710) * chore(deps): bump @sentry/node from 7.33.0 to 7.34.0 (#11711) Bumps [@sentry/node](https://github.com/getsentry/sentry-javascript) from 7.33.0 to 7.34.0. - [Release notes](https://github.com/getsentry/sentry-javascript/releases) - [Changelog](https://github.com/getsentry/sentry-javascript/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-javascript/compare/7.33.0...7.34.0) --- updated-dependencies: - dependency-name: "@sentry/node" dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * fix(route): fix not unique guid and refactor to v2 (#11712) * add null check * add more null check for compensation * remove unused key * refactor: change route * fix typo * add desc text formatting and caching * posting forget to accept * Fix async/await mismatch * Fix async/await mismatch on ctx.cache.tryGet --------- --- lib/v2/linkedin/cn/index.js | 2 +- lib/v2/linkedin/cn/renderer.js | 163 +++++++++++++++++++++++ lib/v2/linkedin/cn/utils.js | 41 +++--- lib/v2/linkedin/templates/cn/posting.art | 4 +- 4 files changed, 188 insertions(+), 22 deletions(-) create mode 100644 lib/v2/linkedin/cn/renderer.js diff --git a/lib/v2/linkedin/cn/index.js b/lib/v2/linkedin/cn/index.js index 341c5f5d3..2958a1472 100644 --- a/lib/v2/linkedin/cn/index.js +++ b/lib/v2/linkedin/cn/index.js @@ -4,7 +4,7 @@ const siteUrl = 'https://www.linkedin.cn/incareer/jobs/search'; module.exports = async (ctx) => { const { title, jobs } = await parseSearchHit(ctx); - const items = await Promise.all(jobs.map((job) => parseJobPosting(job))); + const items = await Promise.all(jobs.map((job) => parseJobPosting(ctx, job))); ctx.state.data = { title: `领英 - ${title}`, link: siteUrl, diff --git a/lib/v2/linkedin/cn/renderer.js b/lib/v2/linkedin/cn/renderer.js new file mode 100644 index 000000000..52509e888 --- /dev/null +++ b/lib/v2/linkedin/cn/renderer.js @@ -0,0 +1,163 @@ +const text_tag = { + LINE_BREAK: 0, + INLINE_CODE: 1, + LEGACY_PUBLISHING_EMPHASIS: 2, + LIST_ITEM: 3, + LIST: 4, + STRIKETHROUGH: 5, + SUBSCRIPT: 6, + SUPERSCRIPT: 7, + UNDERLINE: 8, + BOLD: 9, + ITALIC: 10, + PARAGRAPH: 11, +}; + +class TreeNode { + constructor(attr) { + this.attr = attr; + this.start = attr.start; + this.end = attr.start + attr.length; + this.children = []; + } +} +class Ucs2Text { + constructor(text) { + if (Array.isArray(text)) { + this.string = null; + this.codePoints = text; + } else { + this.string = text; + const k = text.length; + const d = []; + + for (let l, e, m = 0; m < k; ) { + 55296 <= (l = text.charCodeAt(m++)) && 56319 >= l && m < k ? (56320 === (64512 & (e = text.charCodeAt(m++))) ? d.push(((1023 & l) << 10) + (1023 & e) + 65536) : (d.push(l), m--)) : d.push(l); + } + this.codePoints = d; + } + this.length = this.codePoints.length; + } + substring(start, end) { + let _len = this.length, + _start = start, + _end = end; + if (_end === 0) { + return new Ucs2Text(''); + } + if (((isNaN(_start) || 0 > _start) && (_start = 0), (isNaN(_end) || 0 > _end) && (_end = _len), _start > _len && (_start = _len), _end > _len && (_end = _len), _end < _start)) { + _len = [_start, _end]; + _end = _len[0]; + _start = _len[1]; + } + return new Ucs2Text(_start === _end ? '' : this.codePoints.slice(_start, _end)); + } + slice(a, b) { + return this.substring(a, b).toString(); + } + toString() { + return null !== this.string + ? this.string + : this.codePoints + .map((a) => { + let b = ''; + return 65535 < a && ((b += String.fromCharCode((((a -= 65536) >>> 10) & 1023) | 55296)), (a = 1023 & (56320 | a))), b + String.fromCharCode(a); + }) + .join(''); + } +} + +const parseAttr = (description) => { + const { attributes, text } = description; + const attrs = [...attributes]; + attrs.sort((b, d) => { + const f = b.start + b.length, + e = d.start + d.length; + return f !== e ? f - e : text_tag[b.detailData.style] - text_tag[d.detailData.style]; + }); + const n = []; + for (const q of attrs) { + const p = new TreeNode(q); + for (let w = q.start; 0 < n.length && n[n.length - 1].attr.start >= w; ) { + p.children.push(n.pop()); + } + p.children.reverse(); + n.push(p); + } + + const m = new Ucs2Text(text); + const renderSingle = (node) => { + const style = node?.attr?.detailData?.style; + switch (style) { + case 'BOLD': + return `${node.text}`; + case 'LINE_BREAK': + return `
`; + case 'LIST_ITEM': + return `
  • ${node.text}
    • `; + case 'LIST': + return `
      ${node.text}
    `; + case 'INLINE_CODE': + return `${node.text}`; + case 'LEGACY_PUBLISHING_EMPHASIS': + return `${node.text}`; + case 'STRIKETHROUGH': + return `${node.text}`; + case 'SUBSCRIPT': + return `${node.text}`; + case 'SUPERSCRIPT': + return `${node.text}`; + case 'UNDERLINE': + return `${node.text}`; + case 'ITALIC': + return `${node.text}`; + case 'PARAGRAPH': + return `

    ${node.text}

    `; + default: + return node.text || node; + } + }; + + const render = (node) => { + // DFS render + if (node.children.length === 0) { + node.text = m.slice(node.start, node.end); + return renderSingle(node); + } + const res = []; + let s = node.start; + for (const child of node.children) { + if (s < child.start) { + res.push(m.slice(s, child.start)); // added text before 1st child node + } + s = child.end; + res.push(render(child)); + } + + if (s < node.end) { + res.push(m.slice(s, node.end)); // added text after the last child node + } + node.text = res.join(''); + return renderSingle(node); + }; + + const q = []; + let p = 0; + for (let w = 0; w < n.length; w++) { + // BFS render + const e = n[w]; + if (e.start > p) { + q.push(m.slice(p, e.start)); + } + p = e.end; + q.push(render(e)); + } + if (p < m.length) { + q.push(m.slice(p, m.length)); + } + return q.join(''); +}; + +module.exports = { + parseAttr, +}; diff --git a/lib/v2/linkedin/cn/utils.js b/lib/v2/linkedin/cn/utils.js index ec7f148cf..6852f8770 100644 --- a/lib/v2/linkedin/cn/utils.js +++ b/lib/v2/linkedin/cn/utils.js @@ -2,6 +2,7 @@ const crypto = require('crypto'); const path = require('path'); const { art } = require('@/utils/render'); const got = require('@/utils/got'); +const { parseAttr } = require('./renderer'); const apiUrl = 'https://www.linkedin.cn/karpos/api/graphql'; const searchHitQueryId = 'searchSearchHitsByJob.be362cd720abd0ebf89b4bbc3253047f'; @@ -90,27 +91,29 @@ const parseSearchHit = async (ctx) => { }; }; -const parseJobPosting = async (jobPosting) => { +const parseJobPosting = (ctx, jobPosting) => { const entityUrn = jobPosting.entityUrn; - const variables = { - jobPostingUrn: entityUrn, - }; - const resp = await got.post(apiUrl, { - headers: makeHeader(), - body: makeBody({ - operationName: 'jobViewPage', - variables: makeVariables(variables), - queryId: jobPostingQueryId, - }), + return ctx.cache.tryGet(`linkedincn:${entityUrn}`, async () => { + const resp = await got.post(apiUrl, { + headers: makeHeader(), + body: makeBody({ + operationName: 'jobViewPage', + variables: makeVariables({ + jobPostingUrn: entityUrn, + }), + queryId: jobPostingQueryId, + }), + }); + const job = resp.data.data.jobsJobPostingsById; + job.desc = parseAttr(job.description); + return { + title: `${jobPosting.companyName} 正在找 ${jobPosting.title}`, + link: `https://www.linkedin.cn/incareer/jobs/view/${entityUrn.split(':').pop()}`, + guid: `linkedincn:${entityUrn}`, + description: art(path.join(__dirname, '../templates/cn/posting.art'), job), + pubDate: jobPosting.listedAt, + }; }); - const job = resp.data.data.jobsJobPostingsById; - return { - title: `${jobPosting.companyName} 正在找 ${jobPosting.title}`, - link: `https://www.linkedin.cn/incareer/jobs/view/${entityUrn.split(':').pop()}`, - guid: `linkedincn:${entityUrn}`, - description: art(path.join(__dirname, '../templates/cn/posting.art'), job), - pubDate: jobPosting.listedAt, - }; }; module.exports = { diff --git a/lib/v2/linkedin/templates/cn/posting.art b/lib/v2/linkedin/templates/cn/posting.art index e406bfa99..441113989 100644 --- a/lib/v2/linkedin/templates/cn/posting.art +++ b/lib/v2/linkedin/templates/cn/posting.art @@ -19,7 +19,7 @@

    公司介绍

    {{company.name}}

    员工人数：{{company.employeeCount}}

    -

    {{company.localizedDescription}}

    +

    {{company.localizedDescription}}

    {{ /if }}

    职位介绍

    -

    {{description.text}}

    +
    {{@ desc}}