chore(route): support for new dribbble (#13243)
* chore(route): support for new dribbble * chore: date and author support * refactor: migrate to v2 ---------
This commit is contained in:
Nexmoe
GitHub
parent
6c6b22f76f
commit
6fc2c4b500
|
|
@ -66,9 +66,9 @@ router.get('/geektime/news', lazyloadRouteHandler('./routes/geektime/news'));
|
|||
router.get('/infzm/:id', lazyloadRouteHandler('./routes/infzm/news'));
|
||||
|
||||
// Dribbble
|
||||
router.get('/dribbble/popular/:timeframe?', lazyloadRouteHandler('./routes/dribbble/popular'));
|
||||
router.get('/dribbble/user/:name', lazyloadRouteHandler('./routes/dribbble/user'));
|
||||
router.get('/dribbble/keyword/:keyword', lazyloadRouteHandler('./routes/dribbble/keyword'));
|
||||
// router.get('/dribbble/popular/:timeframe?', lazyloadRouteHandler('./routes/dribbble/popular'));
|
||||
// router.get('/dribbble/user/:name', lazyloadRouteHandler('./routes/dribbble/user'));
|
||||
// router.get('/dribbble/keyword/:keyword', lazyloadRouteHandler('./routes/dribbble/keyword'));
|
||||
|
||||
// 虎牙
|
||||
router.get('/huya/live/:id', lazyloadRouteHandler('./routes/huya/live'));
|
||||
|
|
|
|||
|
|
@ -1,94 +0,0 @@
|
|||
const got = require('@/utils/got');
|
||||
const cheerio = require('cheerio');
|
||||
const url = require('url');
|
||||
|
||||
async function load(link) {
|
||||
const response = await got.get(link);
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
const shotMedia = $('.media-shot,.main-shot') // join multiple shots together
|
||||
.toArray()
|
||||
.map((element) => {
|
||||
const object = $(element);
|
||||
|
||||
// remove the content that we don't want to show
|
||||
object.find('span.cropped-indicator').remove();
|
||||
|
||||
return object.html();
|
||||
})
|
||||
.join('');
|
||||
const shotDescription = $('.shot-desc').html() || '';
|
||||
|
||||
const shotLikes = $('.shot-likes').text() || '0 likes';
|
||||
const shotSaves = $('.shot-saves').text() || '0 saves';
|
||||
|
||||
const description = `${shotMedia}<br />
|
||||
${shotDescription}<br />
|
||||
${shotLikes}<br />
|
||||
${shotSaves}`;
|
||||
|
||||
const pubDateString = $('.shot-date').text();
|
||||
const pubDate = new Date(pubDateString).toUTCString();
|
||||
|
||||
return {
|
||||
description,
|
||||
pubDate,
|
||||
};
|
||||
}
|
||||
|
||||
function ProcessFeed(list, caches) {
|
||||
const host = 'https://dribbble.com';
|
||||
|
||||
return Promise.all(
|
||||
list.map(async (item) => {
|
||||
const $ = cheerio.load(item);
|
||||
|
||||
// The link of item is "/signup/new" when access "https://dribbble.com/search/something"
|
||||
// So get url by id
|
||||
const itemId = $(item).attr('id').replace('screenshot-', '');
|
||||
const itemUrl = url.resolve(host, `/shots/${itemId}`);
|
||||
|
||||
const team = $('.attribution-team').text() ? ' for ' + $('.attribution-team a.url').text() : '';
|
||||
const author = 'by ' + $('.shot-byline-user a.url').text();
|
||||
|
||||
const single = {
|
||||
title: $('.shot-title').text(),
|
||||
link: itemUrl,
|
||||
author: author + team,
|
||||
guid: itemUrl,
|
||||
};
|
||||
|
||||
const other = await caches.tryGet(itemUrl, () => load(itemUrl));
|
||||
|
||||
return Promise.resolve(Object.assign({}, single, other));
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
const getData = async (ctx, url, title) => {
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url,
|
||||
headers: {
|
||||
Referer: url,
|
||||
},
|
||||
});
|
||||
|
||||
const data = response.data;
|
||||
|
||||
const $ = cheerio.load(data);
|
||||
const list = $('ol.dribbbles.group > li').get();
|
||||
|
||||
const result = await ProcessFeed(list, ctx.cache);
|
||||
|
||||
return {
|
||||
title,
|
||||
link: url,
|
||||
description: $('meta[name="description"]').attr('content'),
|
||||
item: result,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getData,
|
||||
};
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
module.exports = {
|
||||
'/keyword/:keyword': ['DIYgod', 'loganrockmore'],
|
||||
'/popular/:timeframe?': ['DIYgod', 'loganrockmore'],
|
||||
'/user/:name': ['DIYgod', 'loganrockmore'],
|
||||
};
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
module.exports = {
|
||||
'dribbble.com': {
|
||||
_name: 'Dribbble',
|
||||
'.': [
|
||||
{
|
||||
title: 'Keyword',
|
||||
docs: 'https://docs.rsshub.app/routes/design#dribbble',
|
||||
source: ['/search/shots/recent'],
|
||||
target: (_, url) => `/dribbble/keyword/${new URL(url).searchParams.get('q')}`,
|
||||
},
|
||||
{
|
||||
title: 'Popular',
|
||||
docs: 'https://docs.rsshub.app/routes/design#dribbble',
|
||||
source: ['/'],
|
||||
target: '/dribbble/popular',
|
||||
},
|
||||
{
|
||||
title: 'User (or team)',
|
||||
docs: 'https://docs.rsshub.app/routes/design#dribbble',
|
||||
source: ['/:name'],
|
||||
target: '/dribbble/user/:name',
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
module.exports = (router) => {
|
||||
router.get('/keyword/:keyword', require('./keyword'));
|
||||
router.get('/popular/:timeframe?', require('./popular'));
|
||||
router.get('/user/:name', require('./user'));
|
||||
};
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{{ if shotMedia }}
|
||||
{{@ shotMedia }}<br>
|
||||
{{ /if }}
|
||||
|
||||
{{ if description.length }}
|
||||
{{@ description.html() }}<br>
|
||||
{{ /if }}
|
||||
|
||||
{{ if shotData.likesCount }}
|
||||
{{ shotData.likesCount }} likes<br>
|
||||
{{ /if }}
|
||||
|
||||
{{ if shotData.savesCount }}
|
||||
{{ shotData.savesCount }} saves
|
||||
{{ /if }}
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
const got = require('@/utils/got');
|
||||
const cheerio = require('cheerio');
|
||||
const { parseDate } = require('@/utils/parse-date');
|
||||
const { art } = require('@/utils/render');
|
||||
const { join } = require('path');
|
||||
|
||||
// Refactored function to load a link asynchronously
|
||||
async function load(link) {
|
||||
// Make a GET request to the specified and retrieve the response
|
||||
const response = await got(link);
|
||||
const $ = cheerio.load(response.data);
|
||||
|
||||
const shotData = JSON.parse(
|
||||
$('script')
|
||||
.text()
|
||||
.match(/shotData:\s(\{.+?\}),\n/)[1]
|
||||
);
|
||||
|
||||
// Join multiple shots together by selecting elements with class 'media-shot' or 'main-shot' or 'block-media-wrapper'
|
||||
// 'block-media-wrapper' is a new class of dribbble
|
||||
const shotMedia = $('.media-shot, .main-shot, .block-media-wrapper')
|
||||
.toArray()
|
||||
.map((element) => {
|
||||
const object = $(element);
|
||||
|
||||
// remove the content that we don't want to show
|
||||
object.find('span.cropped-indicator, button').remove();
|
||||
|
||||
object.find('video').each((_, video) => {
|
||||
video = $(video);
|
||||
|
||||
if (!video.attr('src') && video.data('src')) {
|
||||
video.attr('src', video.data('src'));
|
||||
video.removeAttr('data-src');
|
||||
video.removeAttr('data-video-small');
|
||||
video.removeAttr('data-video-medium');
|
||||
video.removeAttr('data-video-large');
|
||||
}
|
||||
});
|
||||
object.find('img').each((_, img) => {
|
||||
img = $(img);
|
||||
|
||||
if (img.data('animated-url')) {
|
||||
img.attr('src', img.data('animated-url'));
|
||||
img.removeAttr('data-animated-url');
|
||||
img.removeAttr('srcset');
|
||||
}
|
||||
|
||||
if (!img.attr('src') && img.data('src')) {
|
||||
img.attr('src', img.data('src').split('?')[0]);
|
||||
img.removeAttr('data-src');
|
||||
}
|
||||
|
||||
img.attr('src', img.attr('src').split('?')[0]);
|
||||
img.removeAttr('srcset');
|
||||
img.removeAttr('data-srcset');
|
||||
});
|
||||
object.find('a').each((_, a) => {
|
||||
a = $(a);
|
||||
a.removeAttr('data-pswp-srcset');
|
||||
});
|
||||
|
||||
return object.html();
|
||||
})
|
||||
.join('');
|
||||
|
||||
const shotDescription = $('.shot-description-container');
|
||||
|
||||
const author = `${shotData.shotUser.name}${shotData.shotUser.team.length ? ` for ${shotData.shotUser.team.name}` : ''}`;
|
||||
const description = art(join(__dirname, 'templates/description.art'), {
|
||||
shotMedia,
|
||||
shotData,
|
||||
description: shotDescription,
|
||||
});
|
||||
|
||||
// Get the text content of the element with class 'shot-date' and convert it to a UTC string representation of a date
|
||||
const pubDate = parseDate(shotData.postedOn);
|
||||
|
||||
// Return an object containing the description and pubDate
|
||||
return {
|
||||
description,
|
||||
pubDate,
|
||||
author,
|
||||
category: shotData.tags,
|
||||
};
|
||||
}
|
||||
|
||||
// Refactored code with comments for clarity
|
||||
|
||||
function ProcessFeed(list, caches) {
|
||||
const host = 'https://dribbble.com';
|
||||
|
||||
// Use Promise.all to process all items in the list asynchronously
|
||||
return Promise.all(
|
||||
list.map((item) => {
|
||||
const $ = cheerio.load(item);
|
||||
|
||||
// The link of item is "/signup/new" when access "https://dribbble.com/search/something"
|
||||
// So get url by id
|
||||
const itemId = $(item).attr('id').replace('screenshot-', '');
|
||||
|
||||
// Construct the full item URL using the host and the item ID
|
||||
const itemUrl = new URL(`/shots/${itemId}`, host).href;
|
||||
|
||||
// Return a Promise that resolves to an object combining the single item data and the additional data
|
||||
return caches.tryGet(itemUrl, async () => {
|
||||
const { description, pubDate, author, category } = await load(itemUrl);
|
||||
|
||||
return {
|
||||
title: $('.shot-title').text(),
|
||||
link: itemUrl,
|
||||
description,
|
||||
pubDate,
|
||||
author,
|
||||
category,
|
||||
};
|
||||
});
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves data from a given URL and processes it.
|
||||
*
|
||||
* @param {Object} ctx - The context object.
|
||||
* @param {string} url - The URL to retrieve data from.
|
||||
* @param {string} title - The title of the data.
|
||||
* @return {Object} - An object containing the retrieved data and metadata.
|
||||
*/
|
||||
const getData = async (ctx, url, title) => {
|
||||
// Make a GET request to the specified URL
|
||||
const response = await got(url);
|
||||
|
||||
// Extract the response data
|
||||
const data = response.data;
|
||||
|
||||
// Load the response data into a cheerio object
|
||||
const $ = cheerio.load(data);
|
||||
// Get all the list items under the 'ol.dribbbles.group' element
|
||||
const list = $('ol.dribbbles.group > li').toArray();
|
||||
|
||||
// Process the list items using the ProcessFeed function
|
||||
const result = await ProcessFeed(list, ctx.cache);
|
||||
|
||||
// Return an object containing the retrieved data and metadata
|
||||
return {
|
||||
title,
|
||||
link: url,
|
||||
description: $('meta[name="description"]').attr('content'),
|
||||
item: result,
|
||||
};
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getData,
|
||||
};
|
||||
Loading…
Reference in New Issue