fix(route/dailypush): author and date (#21110)
This commit is contained in:
+113
-46
@@ -4,14 +4,14 @@ import { load } from 'cheerio';
|
||||
import type { DataItem } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { parseRelativeDate } from '@/utils/parse-date';
|
||||
|
||||
export const BASE_URL = 'https://www.dailypush.dev';
|
||||
|
||||
export interface ArticleItem {
|
||||
title: string;
|
||||
link: string;
|
||||
author?: string;
|
||||
author: DataItem['author'];
|
||||
pubDate?: Date;
|
||||
category?: string[];
|
||||
description?: string;
|
||||
@@ -19,22 +19,81 @@ export interface ArticleItem {
|
||||
dailyPushUrl?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Try to parse text as a date. Returns the Date if parsing succeeds and is valid, undefined otherwise.
|
||||
*/
|
||||
function tryParseAsDate(text: string): Date | undefined {
|
||||
try {
|
||||
const date = parseRelativeDate(text);
|
||||
return Number.isNaN(date.getTime()) ? undefined : date;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract author from article element
|
||||
*/
|
||||
function extractAuthor(article: ReturnType<CheerioAPI>): string | undefined {
|
||||
function extractAuthor(article: ReturnType<CheerioAPI>): DataItem['author'] {
|
||||
const container = article.find('.flex.items-center.gap-3').first();
|
||||
if (container.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Get all content spans (exclude separator spans with '•')
|
||||
const allSpans = container.find('span');
|
||||
const contentSpans: string[] = [];
|
||||
|
||||
for (let i = 0; i < allSpans.length; i++) {
|
||||
const $span = allSpans.eq(i);
|
||||
const text = $span.text().trim();
|
||||
// Skip separator spans (contain only '•' or have separator classes)
|
||||
if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) {
|
||||
contentSpans.push(text);
|
||||
}
|
||||
}
|
||||
|
||||
// Handle different cases based on number of content spans
|
||||
switch (contentSpans.length) {
|
||||
case 3:
|
||||
// Structure: author, date, reading time
|
||||
if (contentSpans[0].includes(',')) {
|
||||
const authors: DataItem['author'] = contentSpans[0].split(',').map((author) => ({
|
||||
name: author.trim(),
|
||||
}));
|
||||
return authors;
|
||||
}
|
||||
return contentSpans[0];
|
||||
case 2: {
|
||||
// Two cases:
|
||||
// 1. date, reading time (no author)
|
||||
// 2. author, date (no reading time)
|
||||
const firstText = contentSpans[0];
|
||||
if (tryParseAsDate(firstText)) {
|
||||
// First is date, so no author
|
||||
break;
|
||||
}
|
||||
// First is author
|
||||
return firstText;
|
||||
}
|
||||
case 1: {
|
||||
// Could be date or author
|
||||
const text = contentSpans[0];
|
||||
if (tryParseAsDate(text)) {
|
||||
return undefined;
|
||||
}
|
||||
return text;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
// Fallback: use the post source as author
|
||||
const sourceSpan = article.find('span.text-xs.font-medium.uppercase').first();
|
||||
if (sourceSpan.length > 0) {
|
||||
return sourceSpan.text().trim();
|
||||
}
|
||||
|
||||
// Fallback: look for author name in the date section
|
||||
const authorDateText = article.find('.flex.items-center.gap-3').first().text();
|
||||
const authorMatch = authorDateText.match(/^([^•]+?)(?:\s*•)/);
|
||||
if (authorMatch && !/\d{4}/.test(authorMatch[1])) {
|
||||
return authorMatch[1].trim();
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -63,52 +122,59 @@ function extractCategories(article: ReturnType<CheerioAPI>, $: CheerioAPI): stri
|
||||
* Extract publication date from article element
|
||||
*/
|
||||
function extractPubDate(article: ReturnType<CheerioAPI>): Date | undefined {
|
||||
const footer = article.find('.flex.items-center.justify-between.gap-4.flex-wrap').first();
|
||||
const authorAndDate = footer.find('.flex.items-center.gap-3.text-xs').first();
|
||||
|
||||
if (authorAndDate.length === 0) {
|
||||
const container = article.find('.flex.items-center.gap-3').first();
|
||||
if (container.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const spans = authorAndDate.find('span');
|
||||
let dateText: string | undefined;
|
||||
// Get all content spans (exclude separator spans with '•')
|
||||
const allSpans = container.find('span');
|
||||
const contentSpans: string[] = [];
|
||||
|
||||
if (spans.length === 3) {
|
||||
// Has author: date is in the third span (index 2)
|
||||
// Structure: <span>Author</span><span>•</span><span>Date</span>
|
||||
dateText = spans.eq(2).text().trim();
|
||||
} else if (spans.length === 1) {
|
||||
// No author: date is in the first span (index 0)
|
||||
// Structure: <span>Date</span>
|
||||
dateText = spans.eq(0).text().trim();
|
||||
}
|
||||
|
||||
if (!dateText) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
try {
|
||||
return parseDate(dateText);
|
||||
} catch {
|
||||
// If parsing fails, try fallback patterns
|
||||
const datePattern = /((?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d+,\s+\d{4})/i;
|
||||
const match = dateText.match(datePattern);
|
||||
if (match && match[1]) {
|
||||
try {
|
||||
return parseDate(match[1]);
|
||||
} catch {
|
||||
// If parsing fails, keep undefined
|
||||
}
|
||||
for (let i = 0; i < allSpans.length; i++) {
|
||||
const $span = allSpans.eq(i);
|
||||
const text = $span.text().trim();
|
||||
// Skip separator spans (contain only '•' or have separator classes)
|
||||
if (text !== '•' && !$span.hasClass('text-slate-300') && !$span.hasClass('dark:text-slate-600')) {
|
||||
contentSpans.push(text);
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
let dateText: string | undefined;
|
||||
|
||||
// Handle different cases based on number of content spans
|
||||
switch (contentSpans.length) {
|
||||
case 3:
|
||||
// Structure: author, date, reading time
|
||||
dateText = contentSpans[1];
|
||||
break;
|
||||
case 2: {
|
||||
// Two cases:
|
||||
// 1. date, reading time (no author)
|
||||
// 2. author, date (no reading time)
|
||||
const firstText = contentSpans[0];
|
||||
dateText = tryParseAsDate(firstText) ? firstText : contentSpans[1];
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
// Could be date or author
|
||||
const text = contentSpans[0];
|
||||
if (tryParseAsDate(text)) {
|
||||
dateText = text;
|
||||
}
|
||||
break;
|
||||
}
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
return dateText ? tryParseAsDate(dateText) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a single article element into an ArticleItem
|
||||
*/
|
||||
function parseArticle(article: ReturnType<CheerioAPI>, $: CheerioAPI, baseUrl: string): ArticleItem | null {
|
||||
function parseArticle(article: ReturnType<CheerioAPI>, $: CheerioAPI, baseUrl: string): (DataItem & ArticleItem) | null {
|
||||
// Find the title link in h2 > a
|
||||
const titleLink = article.find('h2 a[href^="http"]').first();
|
||||
if (titleLink.length === 0) {
|
||||
@@ -135,12 +201,13 @@ function parseArticle(article: ReturnType<CheerioAPI>, $: CheerioAPI, baseUrl: s
|
||||
return {
|
||||
title,
|
||||
link,
|
||||
author: author || undefined,
|
||||
author,
|
||||
pubDate,
|
||||
category: categories.length > 0 ? categories : undefined,
|
||||
description,
|
||||
articleUrl: link,
|
||||
dailyPushUrl,
|
||||
language: 'en',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -162,7 +229,7 @@ export function parseArticles($: CheerioAPI, baseUrl: string): ArticleItem[] {
|
||||
*/
|
||||
export async function enhanceItemsWithSummaries(items: ArticleItem[]): Promise<DataItem[]> {
|
||||
const itemsWithUrl = items.filter((item) => item.dailyPushUrl !== undefined);
|
||||
const itemsWithoutUrl = items.filter((item) => item.dailyPushUrl === undefined);
|
||||
const itemsWithoutUrl: DataItem[] = items.filter((item) => item.dailyPushUrl === undefined);
|
||||
|
||||
const enhancedItems: DataItem[] = await Promise.all(
|
||||
itemsWithUrl.map((item) =>
|
||||
|
||||
Reference in New Issue
Block a user