RSSHub/lib/v2/cw/utils.js

const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
const logger = require('@/utils/logger');
let cookie;

const baseUrl = 'https://www.cw.com.tw';

const pathMap = {
    today: {
        pageUrl: () => '/today',
        limit: 30,
    },
    master: {
        pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`,
        limit: 12,
    },
    sub: {
        pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`,
        limit: 12,
    },
    author: {
        pageUrl: (channel) => `/author/${channel}`,
        limit: 10,
    },
};

const getCookie = async (browser, tryGet) => {
    if (!cookie) {
        cookie = await tryGet('cw:cookie', async () => {
            const page = await browser.newPage();
            await page.setRequestInterception(true);
            page.on('request', (request) => {
                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
            });
            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
            await page.goto(`${baseUrl}/user/get/cookie-bar`, {
                waitUntil: 'domcontentloaded',
            });
            cookie = await getCookies(page);
            await page.close();
            return cookie;
        });
    }
    return cookie;
};

const parsePage = async (path, browser, ctx) => {
    const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`;

    const cookie = await getCookie(browser, ctx.cache.tryGet);
    const page = await browser.newPage();
    await page.setRequestInterception(true);
    page.on('request', (request) => {
        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
    });
    await setCookies(page, cookie, 'cw.com.tw');
    logger.debug(`Requesting ${pageUrl}`);
    await page.goto(pageUrl, {
        waitUntil: 'domcontentloaded',
    });

    const response = await page.evaluate(() => document.documentElement.innerHTML);
    await page.close();
    const $ = cheerio.load(response);

    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit);
    const items = await parseItems(list, browser, ctx.cache.tryGet);

    return { $, items };
};

const parseList = ($, limit) =>
    $('.caption')
        .toArray()
        .map((item) => {
            item = $(item);
            return {
                title: item.find('h3').text(),
                link: item.find('h3 a').attr('href'),
                pubDate: parseDate(item.find('time').text()),
            };
        })
        .slice(0, limit);

const parseItems = (list, browser, tryGet) =>
    Promise.all(
        list.map((item) =>
            tryGet(item.link, async () => {
                const page = await browser.newPage();
                await page.setRequestInterception(true);
                page.on('request', (request) => {
                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
                });
                await setCookies(page, cookie, 'cw.com.tw');
                logger.debug(`Requesting ${item.link}`);
                await page.goto(item.link, {
                    waitUntil: 'domcontentloaded',
                });

                const response = await page.evaluate(() => document.documentElement.innerHTML);
                await page.close();
                const $ = cheerio.load(response);

                const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
                $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
                $('img.lazyload').each((_, img) => {
                    if (img.attribs['data-src']) {
                        img.attribs.src = img.attribs['data-src'];
                        delete img.attribs['data-src'];
                    }
                });

                item.title = $('head title').text();
                item.category = $('meta[name=keywords]').attr('content').split(',');
                item.pubDate = parseDate(meta.datePublished);
                item.author = meta.author.name.replace(',', ' ') || meta.publisher.name;
                item.description = $('.article__head .container').html() + $('.article__content').html();

                return item;
            })
        )
    );

module.exports = {
    baseUrl,
    pathMap,
    getCookie,
    setCookies,
    parsePage,
    parseList,
    parseItems,
};