diff --git a/lib/routes/linkedin/posts.ts b/lib/routes/linkedin/posts.ts new file mode 100644 index 000000000..c5034fe21 --- /dev/null +++ b/lib/routes/linkedin/posts.ts @@ -0,0 +1,64 @@ +import { Route } from '@/types'; +import puppeteer from '@/utils/puppeteer'; +import { load } from 'cheerio'; +import { parseCompanyName, parseCompanyPosts, BASE_URL } from './utils'; +import logger from '@/utils/logger'; + +export const route: Route = { + path: '/company/:company_id/posts', + categories: ['social-media'], + example: '/linkedin/company/google/posts', + parameters: { company_id: "Company's LinkedIn profile ID" }, + description: "Get company's LinkedIn posts by company ID", + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportRadar: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + name: 'Company Posts', + maintainers: ['saifazmi'], + handler: async (ctx) => { + const company_id = ctx.req.param('company_id'); + + // Puppeteer setup + const browser = await puppeteer(); + const page = await browser.newPage(); + await page.setRequestInterception(true); + + page.on('request', (request) => { + request.resourceType() === 'document' ? request.continue() : request.abort(); + }); + + const url = new URL(`${BASE_URL}/company/${company_id}`); + + logger.http(`Requesting ${url.href}`); + await page.goto(url.href, { + waitUntil: 'domcontentloaded', + }); + + const response = await page.content(); + await page.close(); + + const $ = load(response); + const companyName = parseCompanyName($); + const posts = parseCompanyPosts($); + + await browser.close(); + + return { + title: `LinkedIn - ${companyName}'s Posts`, + link: url.href, + description: `This feed gets ${companyName}'s posts from LinkedIn`, + item: posts.map((post) => ({ + title: post.text, + description: post.text, + link: post.link, + pubDate: post.date, + })), + }; + }, +}; diff --git a/lib/routes/linkedin/utils.ts b/lib/routes/linkedin/utils.ts index a91e390cf..33a8bf7ed 100644 --- a/lib/routes/linkedin/utils.ts +++ b/lib/routes/linkedin/utils.ts @@ -1,9 +1,12 @@ import { load } from 'cheerio'; import { Job } from './models'; +import dayjs from 'dayjs'; /** * Constants */ +const BASE_URL = 'https://www.linkedin.com'; + const KEYWORDS_QUERY_KEY = 'keywords'; const JOB_TYPES_QUERY_KEY = 'f_JT'; @@ -123,4 +126,75 @@ const parseRouteParam = (searchParam: string | null): string => { return encodeURIComponent(searchParam.split(',').join('-')); }; -export { parseParamsToSearchParams, parseParamsToString, parseJobDetail, parseJobSearch, parseRouteParam, JOB_TYPES, JOB_TYPES_QUERY_KEY, EXP_LEVELS, EXP_LEVELS_QUERY_KEY, KEYWORDS_QUERY_KEY }; +/** + * Parse company profile page for posts + * Example page: https://www.linkedin.com/company/google/ + * + * @param {Cheerio} $ HTML string of company profile page + * @returns {Array} Array of company posts + */ +function parseCompanyPosts($) { + const posts = $('ul.updates__list > li') + .toArray() // Convert the Cheerio object to a plain array + .map((elem) => { + const elemHtml = $(elem); + const link = elemHtml.find('a.main-feed-card__overlay-link').attr('href'); + const text = elemHtml.find('p.attributed-text-segment-list__content').text().trim(); + const date = parseRelativeShorthandDate(elemHtml.find('time').text().trim()); + + return { link, text, date }; + }); + + return posts; +} + +/** + * Parse company profile page for its name + * Example page: https://www.linkedin.com/company/google/ + * + * @param {Cheerio} $ HTML string of company profile page + * @returns {String} Company name + */ +function parseCompanyName($) { + return $('h1.top-card-layout__title').text().trim(); +} + +/** + * Parse relative date shorthand string into a Date object + * + * @param {String} shorthand The shorthand string representing the date + * @returns {Date|null} The parsed date or null if the format is invalid + */ +function parseRelativeShorthandDate(shorthand) { + const match = shorthand.match(/^(\d+)([wdmyh])$/); + if (!match) { + return null; + } + + const [, amount, unit] = match; + const unitMap = { + w: 'week', + d: 'day', + m: 'month', + y: 'year', + h: 'hour', + }; + + return dayjs().subtract(Number.parseInt(amount), unitMap[unit]); +} + +export { + parseCompanyPosts, + parseCompanyName, + parseParamsToSearchParams, + parseParamsToString, + parseJobDetail, + parseJobSearch, + parseRouteParam, + BASE_URL, + JOB_TYPES, + JOB_TYPES_QUERY_KEY, + EXP_LEVELS, + EXP_LEVELS_QUERY_KEY, + KEYWORDS_QUERY_KEY, +};