diff --git a/lib/v2/layoffs/index.js b/lib/v2/layoffs/index.js index 57152216c..0f4dc9a65 100644 --- a/lib/v2/layoffs/index.js +++ b/lib/v2/layoffs/index.js @@ -1,25 +1,12 @@ const got = require('@/utils/got'); +const cheerio = require('cheerio'); const { parseDate } = require('@/utils/parse-date'); const ROW_COUNT = 100; const WEBSITE_URL = 'https://layoffs.fyi'; -const SOURCE_URL = `https://airtable.com/v0.3/view/viwA14Z1pM69YIsaW/readSharedViewData? -stringifiedObjectParams=%7B%22shouldUseNestedResponseFormat%22%3Atrue%7D& -requestId=reqQcci3rVyhkcAyX&accessPolicy=%7B%22allowedActions%22%3A%5B%7B%22 -modelClassName%22%3A%22view%22%2C%22modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22 -action%22%3A%22readSharedViewData%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22 -modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22getMetadataForPrinting -%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22modelIdSelector -%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22readSignedAttachmentUrls -%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22modelIdSelector%22%3A%22 -rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22action%22%3A%22 -createBoxDocumentSession%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22 -modelIdSelector%22%3A%22rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22 -action%22%3A%22createDocumentPreviewSession%22%7D%5D%2C%22shareId%22%3A%22shrqYt5kSqMzHV9R5%22%2C%22 -applicationId%22%3A%22app1PaujS9zxVGUZ4%22%2C%22generationNumber%22%3A0%2C%22 -expires%22%3A%222023-02-02T00%3A00%3A00.000Z%22%2C%22 -signature%22%3A%22096478e17705ffdac261da07c5b4c025aff6d8c6d86be027533fd248913798de%22%7D`; +const ENTRY_URL = 'https://airtable.com/embed/shrqYt5kSqMzHV9R5/tbl8c8kanuNB6bPYr'; +const AIRTABLE_HOST = 'https://airtable.com'; /** * getMapping can convert either an array or an object @@ -57,12 +44,55 @@ module.exports = async (ctx) => { 'x-time-zone': 'America/Los_Angeles', }; + // Get the latest data source url + let dataSourceUrl = await ctx.cache.get(ENTRY_URL); + let cacheInvalid = false; + let response; + if (!dataSourceUrl) { + cacheInvalid = true; + } else { + // Try to fetch data, + // it may fail due to outdated url + try { + response = await got({ + method: 'get', + url: dataSourceUrl, + headers, + }); + if (response.statusCode >= 400) { + cacheInvalid = true; + } + } catch { + cacheInvalid = true; + } + } + + if (cacheInvalid) { + // Refetch the data source link from entry page + const sourcePage = await got({ + method: 'get', + url: ENTRY_URL, + }); + const $ = cheerio.load(sourcePage.data); + dataSourceUrl = + AIRTABLE_HOST + + $('script') + .text() + .match(/urlWithParams: "(.*?)"/)[1] + .replace(/\\u002F/g, '/'); + + // Cache it again + ctx.cache.set(ENTRY_URL, dataSourceUrl); + + // Refetch the data source + response = await got({ + method: 'get', + url: dataSourceUrl, + headers, + }); + } + // Get data from data source - const response = await got({ - method: 'get', - url: SOURCE_URL, - headers, - }); const data = response.data.data; const table = data.table;