feat(route/ainvest): rework article and news (#21154)
This commit is contained in:
Tony
GitHub
parent
da15d11d1d
commit
a56e6299d6
|
|
@ -18,9 +18,6 @@ updates:
|
|||
# https://github.com/yoimiya-kokomi/Miao-Yunzai/pull/515 https://github.com/zhangfisher/flex-tools/commit/09b565dfe6e2932bb829613ddbe09f6d0acbccd4
|
||||
- dependency-name: art-template
|
||||
versions: ['>=4.13.3']
|
||||
# no longer includes KJUR.crypto.Cipher for RSA
|
||||
- dependency-name: jsrsasign
|
||||
versions: ['>=11.0.0']
|
||||
# pin jsdom to avoid `Error: require() of ES Module` issue caused by parse5 v8
|
||||
# https://github.com/jsdom/jsdom/issues/3959
|
||||
- dependency-name: jsdom
|
||||
|
|
|
|||
|
|
@ -1,8 +1,5 @@
|
|||
import { fetchContentItems } from '@/routes/ainvest/utils';
|
||||
import type { Route } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { decryptAES, encryptAES, getHeaders, randomString } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/article',
|
||||
|
|
@ -19,49 +16,22 @@ export const route: Route = {
|
|||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['ainvest.com/news'],
|
||||
source: ['www.ainvest.com/news/articles-latest/', 'www.ainvest.com'],
|
||||
},
|
||||
],
|
||||
name: 'Latest Article',
|
||||
maintainers: ['TonyRL'],
|
||||
handler,
|
||||
url: 'ainvest.com/news',
|
||||
url: 'www.ainvest.com/news/articles-latest/',
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const key = randomString(16);
|
||||
|
||||
const { data: response } = await got.post('https://api.ainvest.com/gw/socialcenter/v1/edu/article/listArticle', {
|
||||
headers: getHeaders(key),
|
||||
searchParams: {
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
data: encryptAES(
|
||||
JSON.stringify({
|
||||
batch: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 30,
|
||||
startId: null,
|
||||
tags: {
|
||||
in: ['markettrends', 'premarket', 'companyinsights', 'macro'],
|
||||
and: ['web', 'creationplatform'],
|
||||
},
|
||||
}),
|
||||
key
|
||||
),
|
||||
});
|
||||
|
||||
const { data } = JSON.parse(decryptAES(response, key));
|
||||
|
||||
const items = data.map((item) => ({
|
||||
title: item.title,
|
||||
description: item.content,
|
||||
link: item.sourceUrl,
|
||||
pubDate: parseDate(item.postDate, 'x'),
|
||||
category: [item.nickName, ...item.tags.map((tag) => tag.code)],
|
||||
}));
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 5;
|
||||
const items = await fetchContentItems([109], limit);
|
||||
|
||||
return {
|
||||
title: 'AInvest - Latest Articles',
|
||||
link: 'https://www.ainvest.com/news',
|
||||
link: 'https://www.ainvest.com/news/articles-latest/',
|
||||
language: 'en',
|
||||
item: items,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
import { fetchContentItems } from '@/routes/ainvest/utils';
|
||||
import type { Route } from '@/types';
|
||||
import { ViewType } from '@/types';
|
||||
import got from '@/utils/got';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import { decryptAES, getHeaders, randomString } from './utils';
|
||||
|
||||
export const route: Route = {
|
||||
path: '/news',
|
||||
|
|
@ -21,46 +18,23 @@ export const route: Route = {
|
|||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['ainvest.com/news'],
|
||||
source: ['www.ainvest.com/news/'],
|
||||
},
|
||||
],
|
||||
name: 'Latest News',
|
||||
maintainers: ['TonyRL'],
|
||||
handler,
|
||||
url: 'ainvest.com/news',
|
||||
url: 'www.ainvest.com/news/',
|
||||
};
|
||||
|
||||
async function handler(ctx) {
|
||||
const key = randomString(16);
|
||||
|
||||
const { data: response } = await got('https://api.ainvest.com/gw/news_f10/v1/newsFlash/getNewsData', {
|
||||
headers: getHeaders(key),
|
||||
searchParams: {
|
||||
terminal: 'web',
|
||||
tab: 'all',
|
||||
page: 1,
|
||||
size: ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 50,
|
||||
lastId: '',
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
});
|
||||
|
||||
const { data } = JSON.parse(decryptAES(response, key));
|
||||
|
||||
const items = data.content.map((item) => ({
|
||||
title: item.title,
|
||||
description: item.content,
|
||||
link: item.sourceUrl,
|
||||
pubDate: parseDate(item.publishTime, 'x'),
|
||||
category: item.tagList.map((tag) => tag.nameEn),
|
||||
author: item.userInfo.nickname,
|
||||
upvotes: item.likeCount,
|
||||
comments: item.commentCount,
|
||||
}));
|
||||
const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 5;
|
||||
const streamIds = [109, 416, 438, 529, 721, 834, 835];
|
||||
const items = await fetchContentItems(streamIds, limit);
|
||||
|
||||
return {
|
||||
title: 'AInvest - Latest News',
|
||||
link: 'https://www.ainvest.com/news',
|
||||
link: 'https://www.ainvest.com/news/',
|
||||
language: 'en',
|
||||
item: items,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,69 +1,54 @@
|
|||
import crypto from 'node:crypto';
|
||||
import { config } from '@/config';
|
||||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
import CryptoJS from 'crypto-js';
|
||||
import { hextob64, KEYUTIL, KJUR } from 'jsrsasign';
|
||||
const contentStreamUrl = 'https://news.ainvest.com/news-w-ds-hxcmp-content-stream/content_stream/api/stream_item/v1/query_content_stream';
|
||||
const contentPageUrl = 'https://news.ainvest.com/content-page/v1/page';
|
||||
|
||||
const publicKey =
|
||||
'MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCARnxLlrhTK28bEV7s2IROjT73KLSjfqpKIvV8L+Yhe4BrF0Ut4oOH728HZlbSF0C3N0vXZjLAFesoS4v1pYOjVCPXl920Lh2seCv82m0cK78WMGuqZTfA44Nv7JsQMHC3+J6IZm8YD53ft2d8mYBFgKektduucjx8sObe7eRyoQIDAQAB';
|
||||
const normalizeItem = (item) => ({
|
||||
title: item.title,
|
||||
link: item.h5_url,
|
||||
pubDate: parseDate(item.ctime, 'X'),
|
||||
category: item.content_tags.map((tag) => tag.name),
|
||||
author: item.author_name,
|
||||
image: item.cover_image,
|
||||
seoKey: item.seo_key,
|
||||
});
|
||||
|
||||
const randomString = (length: number) => {
|
||||
if (length > 32) {
|
||||
throw new Error('Max length is 32.');
|
||||
}
|
||||
return uuidv4().replaceAll('-', '').slice(0, length);
|
||||
export const fetchContentStream = (streamId, limit) =>
|
||||
cache.tryGet(
|
||||
`ainvest:news:${streamId}:${limit}`,
|
||||
async () => {
|
||||
const response = await ofetch(contentStreamUrl, {
|
||||
query: {
|
||||
lang: 'en',
|
||||
pageSize: limit,
|
||||
stream_id: streamId,
|
||||
page: 1,
|
||||
},
|
||||
});
|
||||
return response.data.list;
|
||||
},
|
||||
config.cache.routeExpire,
|
||||
false
|
||||
);
|
||||
|
||||
export const fetchContentItems = async (streamIds, limit) => {
|
||||
const streams = await Promise.all(streamIds.map((streamId) => fetchContentStream(streamId, limit)));
|
||||
const list = streams.flat().map((item) => normalizeItem(item));
|
||||
|
||||
return Promise.all(
|
||||
list.map((item) =>
|
||||
cache.tryGet(item.link, async () => {
|
||||
const response = await ofetch(`${contentPageUrl}/${item.seoKey}`);
|
||||
const { data } = response;
|
||||
|
||||
item.description = data.pageInfo.structuredContent.map((c) => c.content).join('');
|
||||
item.image = data.contentInfo.coverImage;
|
||||
|
||||
return item;
|
||||
})
|
||||
)
|
||||
);
|
||||
};
|
||||
|
||||
const uuidv4 = () => crypto.randomUUID();
|
||||
|
||||
/**
|
||||
* @param {string} str
|
||||
* @returns {CryptoJS.lib.WordArray}
|
||||
*/
|
||||
const MD5 = (str) => CryptoJS.MD5(str);
|
||||
|
||||
const encryptAES = (data, key) => {
|
||||
if (typeof key === 'string') {
|
||||
key = MD5(key);
|
||||
}
|
||||
return CryptoJS.AES.encrypt(CryptoJS.enc.Utf8.parse(data), key, {
|
||||
mode: CryptoJS.mode.ECB,
|
||||
padding: CryptoJS.pad.Pkcs7,
|
||||
}).toString();
|
||||
};
|
||||
|
||||
const decryptAES = (data, key) => {
|
||||
if (typeof key === 'string') {
|
||||
key = MD5(key);
|
||||
}
|
||||
return CryptoJS.AES.decrypt(data, key, {
|
||||
mode: CryptoJS.mode.ECB,
|
||||
padding: CryptoJS.pad.Pkcs7,
|
||||
}).toString(CryptoJS.enc.Utf8);
|
||||
};
|
||||
|
||||
const encryptRSA = (data) => {
|
||||
// Original code:
|
||||
// var n = new JSEncrypt();
|
||||
// n.setPublicKey(pubKey);
|
||||
// return n.encrypt(message);
|
||||
// Note: Server will reject the public key if it's encrypted using crypto.publicEncrypt().
|
||||
let pubKey = `-----BEGIN PUBLIC KEY-----${publicKey}-----END PUBLIC KEY-----`;
|
||||
pubKey = KEYUTIL.getKey(pubKey);
|
||||
return hextob64(KJUR.crypto.Cipher.encrypt(data, pubKey));
|
||||
};
|
||||
|
||||
const getHeaders = (key) => {
|
||||
const fingerPrint = uuidv4();
|
||||
|
||||
return {
|
||||
'content-type': 'application/json',
|
||||
'ovse-trace': uuidv4(),
|
||||
callertype: 'USER',
|
||||
fingerprint: encryptAES(fingerPrint, MD5(key)),
|
||||
onetimeskey: encryptRSA(key),
|
||||
timestamp: encryptAES(Date.now(), key),
|
||||
referer: 'https://www.ainvest.com/',
|
||||
};
|
||||
};
|
||||
|
||||
export { decryptAES, encryptAES, getHeaders, randomString };
|
||||
|
|
|
|||
Loading…
Reference in New Issue