136 lines
4.1 KiB
JavaScript
136 lines
4.1 KiB
JavaScript
const path = require('path');
|
|
const got = require('@/utils/got');
|
|
const cheerio = require('cheerio');
|
|
const logger = require('@/utils/logger');
|
|
const config = require('@/config').value;
|
|
|
|
const templates = {
|
|
desc: path.join(__dirname, 'templates/desc.art'),
|
|
cover: path.join(__dirname, 'templates/cover.art'),
|
|
embed: path.join(__dirname, 'templates/embed.art'),
|
|
iframe: path.join(__dirname, 'templates/iframe.art'),
|
|
};
|
|
|
|
const resolveUrl = (url, tls = true, forceResolve = false) => {
|
|
if (!url) {
|
|
return '';
|
|
}
|
|
if (url.startsWith('//')) {
|
|
return (tls ? 'https:' : 'http:') + url;
|
|
}
|
|
if (forceResolve && !url.match(/^https?:\/\//)) {
|
|
return (tls ? 'https://' : 'http://') + url;
|
|
}
|
|
return url;
|
|
};
|
|
|
|
const proxyVideo = (url, proxy) => {
|
|
if (!(url && proxy)) {
|
|
return url + '';
|
|
}
|
|
if (proxy.includes('?')) {
|
|
if (!proxy.endsWith('=')) {
|
|
proxy += '=';
|
|
}
|
|
return proxy + encodeURIComponent(url);
|
|
} else {
|
|
if (!proxy.endsWith('/')) {
|
|
proxy += '/';
|
|
}
|
|
return proxy + url;
|
|
}
|
|
};
|
|
|
|
const getOriginAvatar = (url) =>
|
|
resolveUrl(url)
|
|
.replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
|
|
.replace(/~\w+_\d+x\d+/g, '');
|
|
|
|
const extractRenderData = (html) => {
|
|
if (!html) {
|
|
throw Error('Empty html. The request may be filtered by WAF.');
|
|
}
|
|
|
|
const $ = cheerio.load(html);
|
|
const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
|
|
if (!renderData) {
|
|
const title = $('title').text();
|
|
if (title.includes('验证')) {
|
|
throw Error(title);
|
|
} else if ($('#captcha_container').length > 0) {
|
|
throw Error('Captcha required.');
|
|
}
|
|
throw Error('Failed to get render data.');
|
|
} else if (renderData.isSpider) {
|
|
throw Error('The request was considered to be from a spider.');
|
|
} else {
|
|
return renderData;
|
|
}
|
|
};
|
|
|
|
const gotGet = async (url, ua) => {
|
|
const response = await got(url, {
|
|
headers: {
|
|
'User-Agent': ua, // magic!
|
|
},
|
|
});
|
|
return extractRenderData(response.data);
|
|
};
|
|
|
|
const puppeteerGet = async (pageUrl) => {
|
|
const browser = await require('@/utils/puppeteer')();
|
|
const page = await browser.newPage();
|
|
await page.setRequestInterception(true);
|
|
page.on('request', (request) => {
|
|
request.resourceType() === 'document' ? request.continue() : request.abort();
|
|
});
|
|
await page.goto(pageUrl, {
|
|
waitUntil: 'domcontentloaded',
|
|
});
|
|
const html = await page.evaluate(() => document.documentElement.innerHTML);
|
|
await browser.close();
|
|
return extractRenderData(html);
|
|
};
|
|
|
|
const universalGet = async (url, route, ctx) => {
|
|
const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
|
|
const gotFirst = await ctx.cache.tryGet(
|
|
cacheKey,
|
|
() => '1',
|
|
config.cache.contentExpire,
|
|
false // no refresh now
|
|
);
|
|
|
|
const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
|
|
|
|
let data;
|
|
if (gotFirst === '1') {
|
|
try {
|
|
data = await gotGet(url, ua);
|
|
ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
|
|
} catch (e) {
|
|
logger.warn('Failed to get data with got, trying puppeteer: ' + e.message);
|
|
data = await puppeteerGet(url);
|
|
ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
|
|
}
|
|
} else {
|
|
try {
|
|
data = await puppeteerGet(url);
|
|
ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
|
|
} catch (e) {
|
|
logger.warn('Failed to get data with puppeteer, trying got: ' + e.message);
|
|
data = await gotGet(url, ua);
|
|
ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
|
|
}
|
|
}
|
|
return data;
|
|
};
|
|
|
|
module.exports = {
|
|
templates,
|
|
resolveUrl,
|
|
proxyVideo,
|
|
getOriginAvatar,
|
|
universalGet,
|
|
};
|