xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route/douyin/*): magic needed (#11481) 

* fix(route/douyin/*): magic needed

Signed-off-by: Rongrong <i@rong.moe>

* fix: douyin hashtag

* Use Googlebot UA for sub-route 'live'

Signed-off-by: Rongrong <i@rong.moe>

Signed-off-by: Rongrong <i@rong.moe>
This commit is contained in:
Rongrong 2022-12-22 22:36:39 +08:00 committed by GitHub
parent df521a6f2c
commit 58d00a6917
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 109 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
lib/v2/douyin/hashtag.js
View File

@ -1,4 +1,3 @@
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { art } = require('@/utils/render');
const config = require('@/config').value;
@ -23,19 +22,26 @@ module.exports = async (ctx) => {
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
await page.setRequestInterception(true);
let awemeList = '';
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
});
page.on('response', async (response) => {
const request = response.request();
if (request.url().includes('/v1/web/challenge/aweme')) {
awemeList = await response.json();
}
});
await page.goto(tagUrl, {
waitUntil: 'domcontentloaded',
waitUntil: 'networkidle2',
});
await page.waitForSelector('#RENDER_DATA');
const html = await page.evaluate(() => document.documentElement.innerHTML);
const html = await page.evaluate(() => document.getElementById('RENDER_DATA').innerText);
await browser.close();
const $ = cheerio.load(html);
const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
const renderData = JSON.parse(decodeURIComponent(html));
const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail);
renderData[dataKey].defaultData = awemeList.aweme_list;
return renderData[dataKey];
},
config.cache.routeExpire,
@ -49,7 +55,7 @@ module.exports = async (ctx) => {
const posts = tagData.defaultData;
const items = posts.map((post) => {
// parse video
let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
let videoList = post.video && post.video.bit_rate && post.video.bit_rate.map((bit_rate) => resolveUrl(bit_rate.play_addr.url_list[0]));
if (relay) {
videoList = videoList.map((item) => proxyVideo(item, relay));
}
@ -57,14 +63,9 @@ module.exports = async (ctx) => {
duration = duration && duration / 1000;
let img;
// if (!embed) {
// img = post.video && post.video.dynamicCover; // dynamic cover (webp)
// img = post.video && post.video.dynamic_cover && post.video.dynamic_cover.url_list[post.video.dynamic_cover.url_list.length - 1]; // dynamic cover (webp)
// }
img =
img ||
(post.video &&
((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
post.video.originCover || // LD
post.video.cover)); // even more LD
img = img || (post.video && post.video.origin_cover && post.video.origin_cover.url_list[post.video.origin_cover.url_list.length - 1]);
img = img && resolveUrl(img);
// render description
@ -76,10 +77,10 @@ module.exports = async (ctx) => {
return {
title: post.desc,
description,
link: `https://www.douyin.com/video/${post.awemeId}`,
pubDate: parseDate(post.createTime * 1000),
category: post.textExtra.map((extra) => extra.hashtagName),
author: post.authorInfo.nickname,
link: `https://www.douyin.com/video/${post.aweme_id}`,
pubDate: parseDate(post.create_time, 'X'),
category: post.text_extra.map((extra) => extra.hashtag_name),
author: post.author.nickname,
};
});

23
lib/v2/douyin/live.js
View File

@ -1,5 +1,5 @@
const config = require('@/config').value;
const { getOriginAvatar } = require('./utils');
const { getOriginAvatar, universalGet } = require('./utils');
module.exports = async (ctx) => {
const { rid } = ctx.params;
@ -9,26 +9,7 @@ module.exports = async (ctx) => {
const pageUrl = `https://live.douyin.com/${rid}`;
const renderData = await ctx.cache.tryGet(
`douyin:live:${rid}`,
async () => {
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(pageUrl, {
waitUntil: 'domcontentloaded',
});
const data = JSON.parse(decodeURIComponent(await page.evaluate(() => document.querySelector('#RENDER_DATA').innerText)));
browser.close();
return data;
},
config.cache.routeExpire,
false
);
const renderData = await ctx.cache.tryGet(`douyin:live:${rid}`, () => universalGet(pageUrl, 'live', ctx), config.cache.routeExpire, false);
if (renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code !== '0') {
throw `Status code ${renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code}`;

19
lib/v2/douyin/user.js
View File

@ -1,9 +1,8 @@
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { art } = require('@/utils/render');
const config = require('@/config').value;
const { fallback, queryToBoolean } = require('@/utils/readable-social');
const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
const { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet } = require('./utils');
module.exports = async (ctx) => {
const uid = ctx.params.uid;
@ -20,27 +19,13 @@ module.exports = async (ctx) => {
const pageData = await ctx.cache.tryGet(
`douyin:user:${uid}`,
async () => {
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(pageUrl, {
waitUntil: 'domcontentloaded',
});
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
const $ = cheerio.load(html);
const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
const renderData = await universalGet(pageUrl, 'user', ctx);
const dataKey = Object.keys(renderData).filter((key) => renderData[key].user && renderData[key].post)[0];
return renderData[dataKey];
},
config.cache.routeExpire,
false
);
const userInfo = pageData.user.user;
const userNickName = userInfo.nickname;
const userDescription = userInfo.desc;

85
lib/v2/douyin/utils.js
View File

@ -1,4 +1,8 @@
const path = require('path');
const got = require('@/utils/got');
const cheerio = require('cheerio');
const logger = require('@/utils/logger');
const config = require('@/config').value;
const templates = {
desc: path.join(__dirname, 'templates/desc.art'),
@ -42,9 +46,90 @@ const getOriginAvatar = (url) =>
.replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
.replace(/~\w+_\d+x\d+/g, '');
const extractRenderData = (html) => {
if (!html) {
throw Error('Empty html. The request may be filtered by WAF.');
}
const $ = cheerio.load(html);
const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
if (!renderData) {
const title = $('title').text();
if (title.includes('验证')) {
throw Error(title);
} else if ($('#captcha_container').length > 0) {
throw Error('Captcha required.');
}
throw Error('Failed to get render data.');
} else if (renderData.isSpider) {
throw Error('The request was considered to be from a spider.');
} else {
return renderData;
}
};
const gotGet = async (url, ua) => {
const response = await got(url, {
headers: {
'User-Agent': ua, // magic!
},
});
return extractRenderData(response.data);
};
const puppeteerGet = async (pageUrl) => {
const browser = await require('@/utils/puppeteer')();
const page = await browser.newPage();
await page.setRequestInterception(true);
page.on('request', (request) => {
request.resourceType() === 'document' ? request.continue() : request.abort();
});
await page.goto(pageUrl, {
waitUntil: 'domcontentloaded',
});
const html = await page.evaluate(() => document.documentElement.innerHTML);
await browser.close();
return extractRenderData(html);
};
const universalGet = async (url, route, ctx) => {
const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
const gotFirst = await ctx.cache.tryGet(
cacheKey,
() => '1',
config.cache.contentExpire,
false // no refresh now
);
const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
let data;
if (gotFirst === '1') {
try {
data = await gotGet(url, ua);
ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
} catch (e) {
logger.warn('Failed to get data with got, trying puppeteer: ' + e.message);
data = await puppeteerGet(url);
ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
}
} else {
try {
data = await puppeteerGet(url);
ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
} catch (e) {
logger.warn('Failed to get data with puppeteer, trying got: ' + e.message);
data = await gotGet(url, ua);
ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
}
}
return data;
};
module.exports = {
templates,
resolveUrl,
proxyVideo,
getOriginAvatar,
universalGet,
};