xiaoxue_admin
/
RSSHub
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(route): layoffs.fyi (#11849) 

* fix(route): layoffs

* fix(route): layoffs: cache and refetch data source link to avoid invalid data source link

* fix(route): layoff: simplify code and use status code check
This commit is contained in:
BrandNewLifeJackie26 2023-02-16 01:17:17 -08:00 committed by GitHub
parent fbccaec65a
commit 3a778e0401
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 51 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
lib/v2/layoffs/index.js
View File

@ -1,25 +1,12 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const ROW_COUNT = 100;
const WEBSITE_URL = 'https://layoffs.fyi';
const SOURCE_URL = `https://airtable.com/v0.3/view/viwA14Z1pM69YIsaW/readSharedViewData?
stringifiedObjectParams=%7B%22shouldUseNestedResponseFormat%22%3Atrue%7D&
requestId=reqQcci3rVyhkcAyX&accessPolicy=%7B%22allowedActions%22%3A%5B%7B%22
modelClassName%22%3A%22view%22%2C%22modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22
action%22%3A%22readSharedViewData%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22
modelIdSelector%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22getMetadataForPrinting
%22%7D%2C%7B%22modelClassName%22%3A%22view%22%2C%22modelIdSelector
%22%3A%22viwA14Z1pM69YIsaW%22%2C%22action%22%3A%22readSignedAttachmentUrls
%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22modelIdSelector%22%3A%22
rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22action%22%3A%22
createBoxDocumentSession%22%7D%2C%7B%22modelClassName%22%3A%22row%22%2C%22
modelIdSelector%22%3A%22rows%20*%5BdisplayedInView%3DviwA14Z1pM69YIsaW%5D%22%2C%22
action%22%3A%22createDocumentPreviewSession%22%7D%5D%2C%22shareId%22%3A%22shrqYt5kSqMzHV9R5%22%2C%22
applicationId%22%3A%22app1PaujS9zxVGUZ4%22%2C%22generationNumber%22%3A0%2C%22
expires%22%3A%222023-02-02T00%3A00%3A00.000Z%22%2C%22
signature%22%3A%22096478e17705ffdac261da07c5b4c025aff6d8c6d86be027533fd248913798de%22%7D`;
const ENTRY_URL = 'https://airtable.com/embed/shrqYt5kSqMzHV9R5/tbl8c8kanuNB6bPYr';
const AIRTABLE_HOST = 'https://airtable.com';
/**
* getMapping can convert either an array or an object
@ -57,12 +44,55 @@ module.exports = async (ctx) => {
'x-time-zone': 'America/Los_Angeles',
};
// Get the latest data source url
let dataSourceUrl = await ctx.cache.get(ENTRY_URL);
let cacheInvalid = false;
let response;
if (!dataSourceUrl) {
cacheInvalid = true;
} else {
// Try to fetch data,
// it may fail due to outdated url
try {
response = await got({
method: 'get',
url: dataSourceUrl,
headers,
});
if (response.statusCode >= 400) {
cacheInvalid = true;
}
} catch {
cacheInvalid = true;
}
}
if (cacheInvalid) {
// Refetch the data source link from entry page
const sourcePage = await got({
method: 'get',
url: ENTRY_URL,
});
const $ = cheerio.load(sourcePage.data);
dataSourceUrl =
AIRTABLE_HOST +
$('script')
.text()
.match(/urlWithParams: "(.*?)"/)[1]
.replace(/\\u002F/g, '/');
// Cache it again
ctx.cache.set(ENTRY_URL, dataSourceUrl);
// Refetch the data source
response = await got({
method: 'get',
url: dataSourceUrl,
headers,
});
}
// Get data from data source
const response = await got({
method: 'get',
url: SOURCE_URL,
headers,
});
const data = response.data.data;
const table = data.table;