diff --git a/lib/routes/dailypush/all.ts b/lib/routes/dailypush/all.ts new file mode 100644 index 000000000..8e0ed6fa3 --- /dev/null +++ b/lib/routes/dailypush/all.ts @@ -0,0 +1,58 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; + +import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; + +export const route: Route = { + path: '/:sort?', + categories: ['programming'], + example: '/dailypush/latest', + parameters: { + sort: { + description: 'Sort order: `` (trending, default) or `latest`', + default: '', + options: [ + { value: '', label: 'Trending' }, + { value: 'latest', label: 'Latest' }, + ], + }, + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['dailypush.dev/', 'dailypush.dev/latest'], + target: '/', + }, + ], + name: 'All', + maintainers: ['TheGeeKing'], + handler, +}; + +async function handler(ctx) { + const { sort = '' } = ctx.req.param(); + const url = sort ? `${BASE_URL}/${sort}` : BASE_URL; + + const response = await ofetch(url); + const $ = load(response); + + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(list); + + const pageTitle = $('title').text() || 'DailyPush - All'; + + return { + title: pageTitle, + link: url, + item: items, + }; +} diff --git a/lib/routes/dailypush/namespace.ts b/lib/routes/dailypush/namespace.ts new file mode 100644 index 000000000..834195367 --- /dev/null +++ b/lib/routes/dailypush/namespace.ts @@ -0,0 +1,9 @@ +import type { Namespace } from '@/types'; + +export const namespace: Namespace = { + name: 'DailyPush', + url: 'dailypush.dev', + description: 'DailyPush aggregates trending dev articles from 200+ blogs, ranked by community signals from Hacker News, Reddit, X & more.', + lang: 'en', + categories: ['programming'], +}; diff --git a/lib/routes/dailypush/tags.ts b/lib/routes/dailypush/tags.ts new file mode 100644 index 000000000..1c4e8bf58 --- /dev/null +++ b/lib/routes/dailypush/tags.ts @@ -0,0 +1,59 @@ +import { load } from 'cheerio'; + +import type { Route } from '@/types'; +import ofetch from '@/utils/ofetch'; + +import { BASE_URL, enhanceItemsWithSummaries, parseArticles } from './utils'; + +export const route: Route = { + path: '/tag/:tag/:sort?', + categories: ['programming'], + example: '/dailypush/tag/backend/trending', + parameters: { + tag: { description: 'Tag name' }, + sort: { + description: 'Sort order: `trending` (default) or `latest`', + default: 'trending', + options: [ + { value: 'trending', label: 'Trending' }, + { value: 'latest', label: 'Latest' }, + ], + }, + }, + features: { + requireConfig: false, + requirePuppeteer: false, + antiCrawler: false, + supportBT: false, + supportPodcast: false, + supportScihub: false, + }, + radar: [ + { + source: ['dailypush.dev/:tag/trending', 'dailypush.dev/:tag/latest', 'dailypush.dev/:tag'], + target: '/tag/:tag/:sort?', + }, + ], + name: 'Tag', + maintainers: ['TheGeeKing'], + handler, +}; + +async function handler(ctx) { + const { tag, sort = 'trending' } = ctx.req.param(); + const url = `${BASE_URL}/${tag}/${sort}`; + + const response = await ofetch(url); + const $ = load(response); + + const list = parseArticles($, BASE_URL); + const items = await enhanceItemsWithSummaries(list); + + const pageTitle = $('title').text() || `DailyPush - ${tag.charAt(0).toUpperCase() + tag.slice(1)}`; + + return { + title: pageTitle, + link: url, + item: items, + }; +} diff --git a/lib/routes/dailypush/utils.ts b/lib/routes/dailypush/utils.ts new file mode 100644 index 000000000..2198cbf99 --- /dev/null +++ b/lib/routes/dailypush/utils.ts @@ -0,0 +1,192 @@ +import type { CheerioAPI } from 'cheerio'; +import { load } from 'cheerio'; + +import type { DataItem } from '@/types'; +import cache from '@/utils/cache'; +import ofetch from '@/utils/ofetch'; +import { parseDate } from '@/utils/parse-date'; + +export const BASE_URL = 'https://www.dailypush.dev'; + +export interface ArticleItem { + title: string; + link: string; + author?: string; + pubDate?: Date; + category?: string[]; + description?: string; + articleUrl: string; + dailyPushUrl?: string; +} + +/** + * Extract author from article element + */ +function extractAuthor(article: ReturnType): string | undefined { + const sourceSpan = article.find('span.text-xs.font-medium.uppercase').first(); + if (sourceSpan.length > 0) { + return sourceSpan.text().trim(); + } + + // Fallback: look for author name in the date section + const authorDateText = article.find('.flex.items-center.gap-3').first().text(); + const authorMatch = authorDateText.match(/^([^•]+?)(?:\s*•)/); + if (authorMatch && !/\d{4}/.test(authorMatch[1])) { + return authorMatch[1].trim(); + } + + return undefined; +} + +/** + * Extract categories/tags from article element + */ +function extractCategories(article: ReturnType, $: CheerioAPI): string[] { + return article + .find('a[href^="/"]') + .toArray() + .map((tagEl) => { + const tagElement = $(tagEl); + const tagHref = tagElement.attr('href'); + const tagText = tagElement.text().trim(); + + // Skip summary/stats links and navigation + if (tagHref && tagText && !tagHref.includes('article/') && !tagHref.includes('Summary') && tagText.length < 50 && !/^(Summary|stats|About|Tags|Toggle|Trending|Latest|Previous|Next)$/i.test(tagText)) { + return tagText; + } + return null; + }) + .filter((tagText): tagText is string => tagText !== null); +} + +/** + * Extract publication date from article element + */ +function extractPubDate(article: ReturnType): Date | undefined { + const footer = article.find('.flex.items-center.justify-between.gap-4.flex-wrap').first(); + const authorAndDate = footer.find('.flex.items-center.gap-3.text-xs').first(); + + if (authorAndDate.length === 0) { + return undefined; + } + + const spans = authorAndDate.find('span'); + let dateText: string | undefined; + + if (spans.length === 3) { + // Has author: date is in the third span (index 2) + // Structure: Author•Date + dateText = spans.eq(2).text().trim(); + } else if (spans.length === 1) { + // No author: date is in the first span (index 0) + // Structure: Date + dateText = spans.eq(0).text().trim(); + } + + if (!dateText) { + return undefined; + } + + try { + return parseDate(dateText); + } catch { + // If parsing fails, try fallback patterns + const datePattern = /((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+,\s+\d{4})/i; + const match = dateText.match(datePattern); + if (match && match[1]) { + try { + return parseDate(match[1]); + } catch { + // If parsing fails, keep undefined + } + } + } + + return undefined; +} + +/** + * Parse a single article element into an ArticleItem + */ +function parseArticle(article: ReturnType, $: CheerioAPI, baseUrl: string): ArticleItem | null { + // Find the title link in h2 > a + const titleLink = article.find('h2 a[href^="http"]').first(); + if (titleLink.length === 0) { + return null; + } + + const title = titleLink.text().trim(); + const link = titleLink.attr('href'); + + if (!title || !link || link.includes('dailypush.dev')) { + return null; + } + + const author = extractAuthor(article); + const description = article.find('p.text-sm.text-muted-foreground').first().text().trim() || undefined; + const categories = extractCategories(article, $); + + const footer = article.find('.flex.items-center.justify-between.gap-4.flex-wrap').first(); + const summaryLink = footer.find('a[href*="/article/"]').first().attr('href'); + const dailyPushUrl = summaryLink ? `${baseUrl}${summaryLink}` : undefined; + + const pubDate = extractPubDate(article); + + return { + title, + link, + author: author || undefined, + pubDate, + category: categories.length > 0 ? categories : undefined, + description, + articleUrl: link, + dailyPushUrl, + }; +} + +/** + * Parse all articles from the page + */ +export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] { + return $('article') + .toArray() + .map((articleEl) => { + const article = $(articleEl); + return parseArticle(article, $, baseUrl); + }) + .filter((parsed): parsed is ArticleItem => parsed !== null); +} + +/** + * Enhance items with full summaries from dailypush article pages + */ +export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise { + const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined); + const itemsWithoutUrl = items.filter((item) => item.dailyPushUrl === undefined); + + const enhancedItems: DataItem[] = await Promise.all( + itemsWithUrl.map((item) => + cache.tryGet(item.dailyPushUrl!, async () => { + // If we have a dailypush article URL, fetch it for the longer summary + try { + const articleResponse = await ofetch(item.dailyPushUrl!); + const $ = load(articleResponse); + + // Find the longer summary/description on the article page + const summary = $('p.font-ibm-plex-sans.leading-relaxed').first(); + + if (summary.length > 0 && summary.text().trim()) { + item.description = summary.text().trim(); + } + } catch { + // If fetching article page fails, keep the original description + } + + return item; + }) + ) + ); + + // Include items without dailyPushUrl as-is + return [...enhancedItems, ...itemsWithoutUrl]; +}