refactor(route): stdaily (#20313)
* refactor(route): stdaily * fix: replace `sort` with `toSorted`
This commit is contained in:
karasu
GitHub
parent
d6606fc111
commit
d1fc409436
|
|
@ -1,146 +1,108 @@
|
|||
import { Route } from '@/types';
|
||||
|
||||
import type { Data, DataItem, Route } from '@/types';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import cache from '@/utils/cache';
|
||||
import path from 'node:path';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import type { ArticleDetailResponse, ArticleListResponse, DateListResponse, SectionListResponse } from './types';
|
||||
import { art } from '@/utils/render';
|
||||
import got from '@/utils/got';
|
||||
import path from 'node:path';
|
||||
import timezone from '@/utils/timezone';
|
||||
import dayjs from 'dayjs';
|
||||
import { load } from 'cheerio';
|
||||
|
||||
const renderDescription = ({ subtitle, quotation, pics, article }) =>
|
||||
art(path.join(__dirname, 'templates/description.art'), {
|
||||
subtitle,
|
||||
quotation,
|
||||
pics,
|
||||
article,
|
||||
});
|
||||
|
||||
const getPageLength = async (url) => {
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url,
|
||||
});
|
||||
|
||||
const $ = load(response.data);
|
||||
const pageLength = $('.right1 .bmname').children().length;
|
||||
return { pageLength, $ };
|
||||
};
|
||||
|
||||
const getArticleList = ($, paperUrl) => {
|
||||
const pageName = $('.zi .zi-top .banci strong').text();
|
||||
const list = $('.zi-meat ul>li')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const link = $(item).find('a').attr('href');
|
||||
const title = $(item).find('a div').text();
|
||||
return {
|
||||
link: `${paperUrl}/${link}`,
|
||||
title: `[${pageName}] ${title}`,
|
||||
// pubDate,
|
||||
};
|
||||
});
|
||||
|
||||
return list;
|
||||
};
|
||||
const createGetPageArticleList = (paperUrl, page = 1) => {
|
||||
const getPageArticleList = async () => {
|
||||
const currentUrl = `${paperUrl}/node_${page + 1}.htm`;
|
||||
const response = await got({
|
||||
method: 'get',
|
||||
url: currentUrl,
|
||||
});
|
||||
|
||||
const $ = load(response.data);
|
||||
return getArticleList($, paperUrl);
|
||||
};
|
||||
|
||||
return getPageArticleList;
|
||||
};
|
||||
|
||||
const getListArticles = async (list, cache) => {
|
||||
const items = await Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(item.link, async () => {
|
||||
const { data } = await got(item.link);
|
||||
const $ = load(data);
|
||||
|
||||
const quotation = $('.right-meat .yinti').text();
|
||||
const subtitle = $('.right-meat .futi').text();
|
||||
const article = $('.right-meat .tuwen .article #ozoom').html();
|
||||
const pics = $('.right-meat .tuwen .picture')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const pic = {};
|
||||
$(item)
|
||||
.find('tr')
|
||||
.toArray()
|
||||
.map((row) => {
|
||||
const src = $(row).find('img').attr('src');
|
||||
if (src) {
|
||||
pic.src = src;
|
||||
} else {
|
||||
pic.des = $(row).find('td').text();
|
||||
}
|
||||
return null;
|
||||
});
|
||||
return pic;
|
||||
});
|
||||
|
||||
item.author = $('.right-meat .author').text();
|
||||
item.description = renderDescription({ subtitle, quotation, article, pics });
|
||||
|
||||
return item;
|
||||
})
|
||||
)
|
||||
);
|
||||
return items;
|
||||
};
|
||||
const SITE_ID = '811c18b08cf04e79be3b67d6902ee1a7';
|
||||
const CODE = 'KJRB';
|
||||
const API_HOST = 'https://epaper.stdaily.com/stdailynewspaperapi';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/digitalpaper',
|
||||
categories: ['traditional-media'],
|
||||
example: '/stdaily/digitalpaper',
|
||||
parameters: {},
|
||||
features: {
|
||||
requireConfig: false,
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: false,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
name: '科技日报',
|
||||
maintainers: ['lyqluis'],
|
||||
maintainers: ['lyqluis', 'KarasuShin'],
|
||||
handler,
|
||||
features: {
|
||||
supportRadar: true,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['epaper.stdaily.com/statics/technology-site/index.html'],
|
||||
target: '/digitalpaper',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
async function handler() {
|
||||
const date = new Date();
|
||||
const dateStr = date.toLocaleDateString('zh-CN', {
|
||||
year: 'numeric',
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
const dateListResponse = await ofetch<DateListResponse>(`${API_HOST}/uv/article/period/date`, {
|
||||
method: 'POST',
|
||||
body: {
|
||||
code: CODE,
|
||||
date: dayjs(new Date(), 'YYYY-MM-01'),
|
||||
},
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' },
|
||||
});
|
||||
const formatedDate = dateStr.replace('/', '-'); // 'yyyy-mm/dd'
|
||||
|
||||
const rootUrl = 'http://digitalpaper.stdaily.com/http_www.kjrb.com/kjrb/html';
|
||||
const currentUrl = `${rootUrl}/${formatedDate}`;
|
||||
const lastDate = dateListResponse.obj.dateList.toSorted().at(-1);
|
||||
|
||||
// get page
|
||||
const restPageLists = [];
|
||||
const allPageArticleLists = [];
|
||||
const { pageLength, $ } = await getPageLength(`${currentUrl}/node_2.htm`);
|
||||
allPageArticleLists.push(...getArticleList($, currentUrl));
|
||||
for (let i = 2; i <= pageLength; i++) {
|
||||
restPageLists.push(createGetPageArticleList(currentUrl, i));
|
||||
}
|
||||
// get pages' article list
|
||||
allPageArticleLists.push(...(await Promise.all(restPageLists.map((getPageArticleList) => getPageArticleList()))).flat());
|
||||
// get all articles
|
||||
const items = await getListArticles(allPageArticleLists, cache);
|
||||
const sectionListResponse = await ofetch<SectionListResponse>(`${API_HOST}/uv/article/period/periodTime`, {
|
||||
method: 'POST',
|
||||
body: {
|
||||
code: CODE,
|
||||
periodTime: lastDate,
|
||||
siteId: SITE_ID,
|
||||
},
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' },
|
||||
});
|
||||
|
||||
const items = await Promise.all(
|
||||
sectionListResponse.obj.editionList.map((section) =>
|
||||
cache.tryGet(`stdaily:epaper:${section.id}`, async () => {
|
||||
const { list } = await ofetch<ArticleListResponse>(`${API_HOST}/uv/article/article/editionId`, {
|
||||
method: 'POST',
|
||||
body: {
|
||||
id: section.id,
|
||||
siteId: SITE_ID,
|
||||
},
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' },
|
||||
});
|
||||
|
||||
return await Promise.all(
|
||||
list.map(
|
||||
async (article) =>
|
||||
await cache.tryGet(`stddaily:epaper:article:${article.id}`, async () => {
|
||||
const {
|
||||
obj: { articleVo },
|
||||
} = await ofetch<ArticleDetailResponse>(`${API_HOST}/uv/article/article/articleId`, {
|
||||
method: 'POST',
|
||||
body: {
|
||||
code: CODE,
|
||||
id: article.id,
|
||||
siteId: SITE_ID,
|
||||
},
|
||||
headers: { 'X-Requested-With': 'XMLHttpRequest' },
|
||||
});
|
||||
|
||||
return {
|
||||
title: `${articleVo.editionName.match(/：(.*)/)?.[1]} - ${load(articleVo.title).text()}`,
|
||||
description: art(path.join(__dirname, 'templates/description.art'), {
|
||||
subtitle: articleVo.subtitle,
|
||||
pics: articleVo.picList ?? [],
|
||||
content: articleVo.content,
|
||||
}),
|
||||
pubDate: timezone(parseDate(articleVo.periodTime), +8),
|
||||
author: articleVo.author,
|
||||
link: `https://epaper.stdaily.com/statics/technology-site/index.html#/home?isDetail=1¤tNewsId=${article.id}¤tVersionName=${articleVo.editionName}¤tVersion=${Number(section.editionCode)}&timeValue=${articleVo.periodTime}`,
|
||||
} as DataItem;
|
||||
})
|
||||
)
|
||||
);
|
||||
})
|
||||
)
|
||||
);
|
||||
|
||||
return {
|
||||
title: `科技日报`,
|
||||
link: 'http://digitalpaper.stdaily.com',
|
||||
item: items,
|
||||
};
|
||||
title: '中国科技网 - 科技日报',
|
||||
link: 'https://epaper.stdaily.com',
|
||||
item: items.flat(),
|
||||
image: 'https://www.stdaily.com/favicon.ico',
|
||||
} as Data;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,6 @@ import type { Namespace } from '@/types';
|
|||
|
||||
export const namespace: Namespace = {
|
||||
name: '中国科技网',
|
||||
url: 'digitalpaper.stdaily.com',
|
||||
url: 'epaper.stdaily.com',
|
||||
lang: 'zh-CN',
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
{{if quotation}}
|
||||
<h3>{{quotation}}</h3>
|
||||
{{else if subtitle}}
|
||||
{{if subtitle}}
|
||||
<h3>{{subtitle}}</h3>
|
||||
{{/if}}
|
||||
|
||||
{{if pics.length}}
|
||||
{{each pics}}
|
||||
<div>
|
||||
<img src={{$value.src}} />
|
||||
<p>{{$value.des}}</p>
|
||||
<img src={{$value.url}} />
|
||||
{{if $value.picText }}
|
||||
<p>{{$value.picText}}</p>
|
||||
{{/if }}
|
||||
</div>
|
||||
{{/each}}
|
||||
{{/if}}
|
||||
|
||||
{{@article}}
|
||||
{{@content}}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
export interface DateListResponse {
|
||||
obj: {
|
||||
dateList: string[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface SectionListResponse {
|
||||
obj: {
|
||||
editionList: {
|
||||
editionName: string;
|
||||
periodId: string;
|
||||
editionCode: string;
|
||||
id: string;
|
||||
}[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface ArticleListResponse {
|
||||
list: {
|
||||
id: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface ArticleDetailResponse {
|
||||
obj: {
|
||||
articleVo: {
|
||||
title: string;
|
||||
subtitle: string;
|
||||
content: string;
|
||||
editionName: string;
|
||||
picList:
|
||||
| null
|
||||
| {
|
||||
url: string;
|
||||
picText: string;
|
||||
}[];
|
||||
periodTime: string;
|
||||
author: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
Loading…
Reference in New Issue