diff --git a/.gitignore b/.gitignore index cc790af67..2cbd2bc18 100644 --- a/.gitignore +++ b/.gitignore @@ -7,8 +7,7 @@ package-lock.json .idea .DS_Store docs/.vuepress/dist - config/app.json config/config.js - yarn-error.log +tmp diff --git a/.prettierignore b/.prettierignore index f0ad55e7d..06bb38b53 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,4 @@ package.json docs/.vuepress/dist package-lock.json +routes/douyin/fuck-byted-acrawler.js diff --git a/cache/.gitkeep b/cache/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/README.md b/docs/README.md index f157bdd68..c26d75758 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2674,3 +2674,15 @@ GitHub 官方也提供了一些 RSS: 举例: 路由: `/xkcd/what-if` + +## 抖音 + +### 用户动态 + +举例: + +路由: `/douyin/user/:id` + +参数: + +- id, 用户 id, 可在 分享出去获得的用户主页 URL 中找到 diff --git a/package.json b/package.json index 8897c8121..85ee18c20 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "luxon": "^1.3.3", "markdown-it": "^8.4.2", "path-to-regexp": "2.4.0", + "puppeteer": "^1.7.0", "raven": "^2.6.3", "readall": "1.0.0", "redis": "2.8.0", diff --git a/router.js b/router.js index 5f17c0759..37c71d50d 100644 --- a/router.js +++ b/router.js @@ -525,4 +525,7 @@ router.get('/xkcd/what-if', require('./routes/xkcd/what-if')); // Minecraft CurseForge router.get('/curseforge/files/:project', require('./routes/curseforge/files')); +// 抖音 +router.get('/douyin/user/:id', require('./routes/douyin/user')); + module.exports = router; diff --git a/routes/douyin/user.js b/routes/douyin/user.js new file mode 100644 index 000000000..0fa49c7ae --- /dev/null +++ b/routes/douyin/user.js @@ -0,0 +1,65 @@ +const cheerio = require('cheerio'); + +module.exports = async (ctx) => { + const id = ctx.params.id; + const getPage = await require('../../utils/puppeteer'); + const page = await getPage(); + + const timerPromise = new Promise((resolve) => setTimeout(resolve, 1000)); + await timerPromise; + + const data = await new Promise((resolve) => { + const result = { + name: null, + description: null, + list: null, + }; + + page.goto(`https://www.douyin.com/share/user/${id}`); + + page.on('response', (response) => { + const req = response.request(); + if (req.url().match(`www.douyin.com/share/user/${id}`)) { + response.text().then((text) => { + const $ = cheerio.load(text); + result.name = $('.nickname').text(); + result.description = $('.signature').text(); + if (result.list) { + resolve(result); + } + }); + } else if (req.url().match('www.douyin.com/aweme/v1/aweme/post')) { + response.json().then((data) => { + result.list = data; + if (result.name) { + resolve(result); + } + }); + } + }); + }); + + ctx.state.data = { + title: `${data.name}的抖音`, + link: `https://www.douyin.com/share/user/${id}`, + description: data.description, + item: + data.list && + data.list.aweme_list.map((item) => ({ + title: item.share_info.share_desc, + description: ` +

${item.share_info.share_desc}

+

APP 内打开

+

+

`, + link: item.video.play_addr.url_list[0], + })), + }; +}; diff --git a/routes/telegram/stickerpack.js b/routes/telegram/stickerpack.js index ea1f14cfe..7be28fa6f 100644 --- a/routes/telegram/stickerpack.js +++ b/routes/telegram/stickerpack.js @@ -39,7 +39,7 @@ module.exports = async (ctx) => { responseType: 'arraybuffer', }); - const filePath = `cache/${item.file_id}.png`; + const filePath = `tmp/${item.file_id}.png`; await sharp(fileResponse.data) .png() diff --git a/utils/puppeteer.js b/utils/puppeteer.js new file mode 100644 index 000000000..55476f6fb --- /dev/null +++ b/utils/puppeteer.js @@ -0,0 +1,26 @@ +const puppeteer = require('puppeteer'); +const logger = require('./logger'); +const config = require('../config'); + +const options = { + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certifcate-errors', '--ignore-certifcate-errors-spki-list', `--user-agent=${config.ua}`], + headless: true, + ignoreHTTPSErrors: true, + userDataDir: './tmp', +}; + +module.exports = (async () => { + const browser = await puppeteer.launch(options); + logger.info('Puppeteer launched.'); + + return async () => { + const page = await browser.newPage(); + + // 防止 page 未正确关闭,一分钟后自行关闭 + setTimeout(() => { + page.close(); + }, 60000); + + return page; + }; +})(); diff --git a/yarn.lock b/yarn.lock index a1a1c352e..660c60b09 100644 --- a/yarn.lock +++ b/yarn.lock @@ -877,6 +877,12 @@ acorn@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.1.tgz#f095829297706a7c9776958c0afc8930a9b9d9d8" +agent-base@^4.1.0: + version "4.2.1" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-4.2.1.tgz#d89e5999f797875674c07d87f260fc41e83e8ca9" + dependencies: + es6-promisify "^5.0.0" + agentkeepalive@^2.2.0: version "2.2.0" resolved "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-2.2.0.tgz#c5d1bd4b129008f1163f236f86e5faea2026e2ef" @@ -1872,7 +1878,7 @@ concat-map@0.0.1: version "0.0.1" resolved "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" -concat-stream@^1.5.0: +concat-stream@1.6.2, concat-stream@^1.5.0: version "1.6.2" resolved "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz#904bdf194cd3122fc675c77fc4ac3d4ff0fd1a34" dependencies: @@ -2242,7 +2248,7 @@ debug@0.7.4: version "0.7.4" resolved "https://registry.npmjs.org/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39" -debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.1, debug@^2.6.3, debug@^2.6.8: +debug@2.6.9, debug@^2.1.2, debug@^2.2.0, debug@^2.3.3, debug@^2.6.1, debug@^2.6.3, debug@^2.6.8: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" dependencies: @@ -2629,7 +2635,7 @@ es6-promise@^3.2.1: version "3.3.1" resolved "https://registry.npmjs.org/es6-promise/-/es6-promise-3.3.1.tgz#a08cdde84ccdbf34d027a1451bc91d4bcd28a613" -es6-promise@^4.1.0: +es6-promise@^4.0.3, es6-promise@^4.1.0: version "4.2.4" resolved "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.4.tgz#dc4221c2b16518760bd8c39a52d8f356fc00ed29" @@ -2639,6 +2645,12 @@ es6-promisify@^4.1.0: dependencies: es6-promise "^3.2.1" +es6-promisify@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/es6-promisify/-/es6-promisify-5.0.0.tgz#5109d62f3e56ea967c4b63505aef08291c8a5203" + dependencies: + es6-promise "^4.0.3" + es6-symbol@^3.1.1, es6-symbol@~3.1.1: version "3.1.1" resolved "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.1.tgz#bf00ef4fdab6ba1b46ecb7b629b4c7ed5715cc77" @@ -2933,6 +2945,15 @@ extglob@^2.0.4: snapdragon "^0.8.1" to-regex "^3.0.1" +extract-zip@^1.6.6: + version "1.6.7" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-1.6.7.tgz#a840b4b8af6403264c8db57f4f1a74333ef81fe9" + dependencies: + concat-stream "1.6.2" + debug "2.6.9" + mkdirp "0.5.1" + yauzl "2.4.1" + extsprintf@1.3.0: version "1.3.0" resolved "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -2980,6 +3001,12 @@ fastparse@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/fastparse/-/fastparse-1.1.1.tgz#d1e2643b38a94d7583b479060e6c4affc94071f8" +fd-slicer@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" + dependencies: + pend "~1.2.0" + fecha@^2.3.3: version "2.3.3" resolved "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz#948e74157df1a32fd1b12c3a3c3cdcb6ec9d96cd" @@ -3625,6 +3652,13 @@ https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" +https-proxy-agent@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-2.2.1.tgz#51552970fa04d723e04c56d04178c3f92592bbc0" + dependencies: + agent-base "^4.1.0" + debug "^3.1.0" + iconv-lite@0.4.24: version "0.4.24" resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b" @@ -4989,7 +5023,7 @@ mkdirp@0.3.0: version "0.3.0" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e" -mkdirp@0.5.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: +mkdirp@0.5.1, mkdirp@0.5.x, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.1" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903" dependencies: @@ -5582,6 +5616,10 @@ pbkdf2@^3.0.3: safe-buffer "^5.0.1" sha.js "^2.4.8" +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + performance-now@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" @@ -6019,6 +6057,10 @@ promise-inflight@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz#98472870bf228132fcbdd868129bad12c3c029e3" +proxy-from-env@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.0.0.tgz#33c50398f70ea7eb96d21f7b817630a55791c7ee" + prr@~1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz#d3fc114ba06995a45ec6893f484ceb1d78f5f476" @@ -6087,6 +6129,19 @@ punycode@^1.2.4, punycode@^1.4.1: version "1.4.1" resolved "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" +puppeteer@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-1.7.0.tgz#edcba2300a50847202c0f19fd15e7a96171ff3bd" + dependencies: + debug "^3.1.0" + extract-zip "^1.6.6" + https-proxy-agent "^2.2.1" + mime "^2.0.3" + progress "^2.0.0" + proxy-from-env "^1.0.0" + rimraf "^2.6.1" + ws "^5.1.1" + q@^1.0.1, q@^1.1.2: version "1.5.1" resolved "https://registry.npmjs.org/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -8010,6 +8065,12 @@ ws@^4.0.0: async-limiter "~1.0.0" safe-buffer "~5.1.0" +ws@^5.1.1: + version "5.2.2" + resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f" + dependencies: + async-limiter "~1.0.0" + xdg-basedir@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/xdg-basedir/-/xdg-basedir-3.0.0.tgz#496b2cc109eca8dbacfe2dc72b603c17c5870ad4" @@ -8047,6 +8108,12 @@ yargs-parser@^10.0.0: dependencies: camelcase "^4.1.0" +yauzl@2.4.1: + version "2.4.1" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.4.1.tgz#9528f442dab1b2284e58b4379bb194e22e0c4005" + dependencies: + fd-slicer "~1.0.1" + ylru@^1.2.0: version "1.2.1" resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f"