feat(route): 为oreno3d路由添加最大抓取页面功能 (#10549)
* refactor🎨: fix * fix🐛: remove useless visiting * fix🐛: fix bugs * fix docs * docs📝: fix docs bugs * Update docs/en/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Update docs/en/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Update docs/en/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Update docs/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Update docs/en/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * Update docs/en/anime.md Co-authored-by: Tony <TonyRL@users.noreply.github.com> * docs📝: fIX DOCS BUGS
This commit is contained in:
xueli-sherryli
GitHub
parent
b1af9b812d
commit
c6d5bc12b9
|
|
@ -427,29 +427,29 @@ Sources
|
|||
|
||||
### 关键词搜索
|
||||
|
||||
<Route author="xueli-sherryli" example="/oreno3d/search/bronya/latest" path="/oreno3d/search/:keyword/:sort?" :paramsDesc="['关键词', '排序方法，见下表，默认为 `latest`']">
|
||||
<Route author="xueli-sherryli" example="/oreno3d/search/bronya/latest/1" path="/oreno3d/search/:keyword/:sort/:pagelimit?" :paramsDesc="['关键词', '排序方法，见下表', '最大爬取页面数量，默认为 1']" radar="1" rssbud="1">
|
||||
|
||||
| 高評価 | 急上昇 | 新着 | 人気 |
|
||||
| --------- | --- | ------ | ---------- |
|
||||
| favorites | hot | latest | popularity |
|
||||
| 高評価 | 急上昇 | 新着 | 人気 |
|
||||
| --------- | ------ | ------ | ---------- |
|
||||
| favorites | hot | latest | popularity |
|
||||
|
||||
</Route>
|
||||
|
||||
### 角色搜索
|
||||
|
||||
<Route author="xueli-sherryli" example="/oreno3d/characters/283/latest" path="/oreno3d/characters/:characterid/:sort?" :paramsDesc="['角色 id，可在浏览器地址栏找到', '排序方法，见上表，默认为 `latest`']" />
|
||||
<Route author="xueli-sherryli" example="/oreno3d/characters/283/latest/1" path="/oreno3d/characters/:characterid/:sort/:pagelimit?" :paramsDesc="['角色id, 可在浏览器地址栏找到','排序方法，见上表','最大爬取页面数量, 默认为1, 超出则为最大值']" />
|
||||
|
||||
### 作者搜索
|
||||
|
||||
<Route author="xueli-sherryli" example="/oreno3d/authors/3189/latest" path="/oreno3d/authors/:authorid/:sort?" :paramsDesc="['作者 id，可在浏览器地址栏找到', '排序方法，见上表，默认为 `latest`']" />
|
||||
<Route author="xueli-sherryli" example="/oreno3d/authors/3189/latest/1" path="/oreno3d/authors/:authorid/:sort/:pagelimit?" :paramsDesc="['作者id，可在浏览器地址栏找到','排序方法，见上表','最大爬取页面数量,默认为1,超出则为最大值']" radar="1" rssbud="1" />
|
||||
|
||||
### 标签搜索
|
||||
|
||||
<Route author="xueli-sherryli" example="/oreno3d/tags/177/latest" path="/oreno3d/tags/:tagid/:sort?" :paramsDesc="['标签 id，可在浏览器地址栏找到', '排序方法，见上表，默认为 `latest`']" />
|
||||
<Route author="xueli-sherryli" example="/oreno3d/tags/177/latest/1" path="/oreno3d/tags/:tagid/:sort/:pagelimit?" :paramsDesc="['标签 id，可在浏览器地址栏找到','排序方法，见上表','最大爬取页面数量,默认为1,超出则为最大值']" radar="1" rssbud="1" />
|
||||
|
||||
### 原作搜索
|
||||
|
||||
<Route author="xueli-sherryli" example="/oreno3d/origins/3/latest" path="/oreno3d/origins/:originid/:sort?" :paramsDesc="['原作 id，可在浏览器地址栏找到', '排序方法，见上表，默认为 `latest`']" />
|
||||
<Route author="xueli-sherryli" example="/oreno3d/origins/3/latest/1" path="/oreno3d/origins/:originid/:sort/:pagelimit?" :paramsDesc="['原作 id，可在浏览器地址栏找到','排序方法，见上表','最大爬取页面数量,默认为1，超出则为最大值']" radar="1" rssbud="1" />
|
||||
|
||||
## 包子漫画
|
||||
|
||||
|
|
|
|||
|
|
@ -134,26 +134,26 @@ You can use some RSS parsing libraries (like `feedpraser` in `Python`) to receiv
|
|||
|
||||
### Keyword Search
|
||||
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/search/bronya/latest" path="/oreno3d/search/:keyword/:sort?" :paramsDesc="['Search keyword', 'Sort method, see the table below, `latest` by default']" radar="1" rssbud="1">
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/search/bronya/latest/1" path="/oreno3d/search/:keyword/:sort/:pagelimit?" :paramsDesc="['Search keyword', 'Sort method, see the table below', 'The maximum number of pages to be crawled, the default is 1']" radar="1" rssbud="1">
|
||||
|
||||
| favorites | hot | latest | popularity |
|
||||
| --------- | ------ | ------ | ---------- |
|
||||
| favorites | hot | latest | popularity |
|
||||
| favorites | hot | latest | popularity |
|
||||
| --------- | ---- | ------ | ---------- |
|
||||
| favorites | hot | latest | popularity |
|
||||
|
||||
</RouteEn>
|
||||
|
||||
### Character Search
|
||||
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/characters/283/latest" path="/oreno3d/characters/:characterid/:sort?" :paramsDesc="['Character id, can be found in URL' , 'Sort method, see the table above, `latest` by default']" radar="1" rssbud="1"/>
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/characters/283/latest/1" path="/oreno3d/characters/:characterid/:sort/:pagelimit?" :paramsDesc="['Character id, can be found in URL' , 'Sort method, see the table above', 'The maximum number of pages to be crawled, the default is 1']" radar="1" rssbud="1"/>
|
||||
|
||||
### Author Search
|
||||
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/authors/3189/latest" path="/oreno3d/authors/:authorid/:sort?" :paramsDesc="['Author id, can be found in URL' , 'Sort method, see the table above, `latest` by default']" radar="1" rssbud="1"/>
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/authors/3189/latest/1" path="/oreno3d/authors/:authorid/:sort/:pagelimit?" :paramsDesc="['Author id, can be found in URL' , 'Sort method, see the table above', 'The maximum number of pages to be crawled, the default is 1']" radar="1" rssbud="1"/>
|
||||
|
||||
### Tags Search
|
||||
|
||||
<RouteEn tag="xueli-sherryli" example="/oreno3d/tags/177/latest" path="/oreno3d/tags/:tagid/:sort?" :paramsDesc="['Tag id, can be found in URL', 'Sort method, see the table above, `latest` by default']" radar="1" rssbud="1"/>
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/tags/177/latest/1" path="/oreno3d/tags/:tagid/:sort/:pagelimit?" :paramsDesc="['Tag id, can be found in URL', 'Sort method, see the table above', 'The maximum number of pages to be crawled, the default is 1']" radar="1" rssbud="1"/>
|
||||
|
||||
### Origins Search
|
||||
|
||||
<RouteEn origin="xueli-sherryli" example="/oreno3d/origins/3/latest" path="/oreno3d/origins/:originid/:sort?" :paramsDesc="['Origin id, can be found in URL' , 'Sort method, see the table above, `latest` by default']" radar="1" rssbud="1"/>
|
||||
<RouteEn author="xueli-sherryli" example="/oreno3d/origins/3/latest/1" path="/oreno3d/origins/:originid/:sort/:pagelimit?" :paramsDesc="['Origin id, can be found in URL' , 'Sort method, see the table above', 'The maximum number of pages to be crawled, the default is 1']" radar="1" rssbud="1"/>
|
||||
|
|
@ -2,7 +2,7 @@ const got = require('@/utils/got');
|
|||
const { art } = require('@/utils/render');
|
||||
const path = require('path');
|
||||
const cheerio = require('cheerio');
|
||||
const get_sec_page_data = require('./get_sec_page_data');
|
||||
const get_sec_page_data = require("./get_sec_page_data");
|
||||
|
||||
const rootUrl = 'https://oreno3d.com';
|
||||
|
||||
|
|
@ -14,81 +14,143 @@ const sortRename = {
|
|||
};
|
||||
|
||||
function get_user_url(rootUrl, ctx, sort) {
|
||||
let userUrl = '';
|
||||
let userUrl = "";
|
||||
// 判断userurl获取位置
|
||||
if (ctx.params.keyword) {
|
||||
const keyword = ctx.params.keyword;
|
||||
userUrl = `${rootUrl}/search?sort=${sort}&keyword=${keyword}`;
|
||||
} else if (ctx.params.characterid) {
|
||||
}
|
||||
else if (ctx.params.characterid) {
|
||||
const characterid = ctx.params.characterid;
|
||||
userUrl = `${rootUrl}/characters/${characterid}?sort=${sort}`;
|
||||
} else if (ctx.params.authorid) {
|
||||
}
|
||||
else if (ctx.params.authorid) {
|
||||
const authorid = ctx.params.authorid;
|
||||
userUrl = `${rootUrl}/authors/${authorid}?sort=${sort}`;
|
||||
} else if (ctx.params.tagid) {
|
||||
}
|
||||
else if (ctx.params.tagid) {
|
||||
const tagid = ctx.params.tagid;
|
||||
userUrl = `${rootUrl}/tags/${tagid}?sort=${sort}`;
|
||||
} else if (ctx.params.originid) {
|
||||
}
|
||||
else if (ctx.params.originid) {
|
||||
const originid = ctx.params.originid;
|
||||
userUrl = `${rootUrl}/origins/${originid}?sort=${sort}`;
|
||||
}
|
||||
return userUrl;
|
||||
}
|
||||
const selector = 'a.box';
|
||||
module.exports = async (ctx) => {
|
||||
// 下载
|
||||
const sort = ctx.params.sort ?? 'latest';
|
||||
const userUrl = get_user_url(rootUrl, ctx, sort);
|
||||
const response = await got(userUrl);
|
||||
const $ = cheerio.load(response.data);
|
||||
// 第一页，获取搜索主标题
|
||||
const title = $('div.g-main-list').find('h1.main-h').text(); // 搜索标题;//第二页跳转链接
|
||||
|
||||
const list = $(selector);
|
||||
const items = await Promise.all(
|
||||
list
|
||||
.map((_, item) => {
|
||||
// 获取视频链接列表及标题
|
||||
function getLinksTitle(response) {
|
||||
const selector = 'a.box';
|
||||
const $ = cheerio.load(response.data);
|
||||
const title = $("div.g-main-list").find("h1.main-h").text();
|
||||
const list = $(selector);
|
||||
return {title, list};
|
||||
}
|
||||
// 数据爬取和分析
|
||||
async function getData(response) {
|
||||
const $ = cheerio.load(response.data);
|
||||
// 第一页，获取搜索主标题
|
||||
const title = getLinksTitle(response).title;
|
||||
const list = getLinksTitle(response).list;
|
||||
const items = await Promise.all(
|
||||
list.map(async(_, item) => {
|
||||
// 第一页，获取搜各视频地址
|
||||
const link = $(item).attr('href');
|
||||
return ctx.cache.tryGet(link, async () => {
|
||||
// 第二页数据分析
|
||||
const sec_data = await get_sec_page_data(link);
|
||||
// 传递
|
||||
const raw_pic_link = sec_data.raw_pic_link;
|
||||
const video_name = sec_data.video_name;
|
||||
const authors = sec_data.authors;
|
||||
const origins = sec_data.origins;
|
||||
const characters = sec_data.characters;
|
||||
const tags = sec_data.tags;
|
||||
const desc = sec_data.desc;
|
||||
const iwara_link = sec_data.iwara_link;
|
||||
const oreno3d_link = sec_data.oreno3d_link;
|
||||
// 打包
|
||||
return {
|
||||
title: `${video_name} - ${authors}`,
|
||||
author: authors,
|
||||
// 第二页数据分析+缓存
|
||||
const sec_data = await get_sec_page_data(link);
|
||||
// 传递
|
||||
const raw_pic_link = sec_data.raw_pic_link;
|
||||
const video_name = sec_data.video_name;
|
||||
const authors = sec_data.authors;
|
||||
const origins = sec_data.origins;
|
||||
const characters = sec_data.characters;
|
||||
const tags = sec_data.tags;
|
||||
const desc = sec_data.desc;
|
||||
const iwara_link = sec_data.iwara_link;
|
||||
const oreno3d_link = sec_data.oreno3d_link;
|
||||
// 打包,缓存HTML
|
||||
const description = art(path.join(__dirname, 'templates/description.art'), {
|
||||
raw_pic_link,
|
||||
video_name,
|
||||
authors,
|
||||
origins,
|
||||
characters,
|
||||
tags,
|
||||
desc,
|
||||
iwara_link,
|
||||
oreno3d_link
|
||||
});
|
||||
const title = `${video_name} - ${authors}`;
|
||||
const realData = await ctx.cache.tryGet(oreno3d_link, () => {
|
||||
const result = {
|
||||
title,
|
||||
author:authors,
|
||||
link: oreno3d_link,
|
||||
category: tags.split(' '),
|
||||
description: art(path.join(__dirname, 'templates/description.art'), {
|
||||
raw_pic_link,
|
||||
video_name,
|
||||
authors,
|
||||
origins,
|
||||
characters,
|
||||
tags,
|
||||
desc,
|
||||
iwara_link,
|
||||
oreno3d_link,
|
||||
}),
|
||||
description
|
||||
};
|
||||
return result;
|
||||
});
|
||||
})
|
||||
.get()
|
||||
return realData;
|
||||
}).get(),
|
||||
);
|
||||
|
||||
return {items, title};
|
||||
}
|
||||
// 参数定义
|
||||
const sort = ctx.params.sort;
|
||||
let pagelimit = ctx.params.pagelimit ?? 1;
|
||||
// 根链接
|
||||
const userUrl = get_user_url(rootUrl, ctx, sort);
|
||||
// 确认最大页数并比较修改(同时也是获取第一页的内容)
|
||||
const response = await got(userUrl);
|
||||
const $ = cheerio.load(response.data);
|
||||
const maxPageSelector = "div.container > main > div.g-main-list > ul.pagination > li:last-child > a";
|
||||
// 页码判断
|
||||
if ($(maxPageSelector)) {
|
||||
const pageLink = new URLSearchParams($(maxPageSelector).attr('href'));
|
||||
const actualNum = pageLink.get("page");// 获取最大页数
|
||||
if (parseInt(pagelimit) >= parseInt(actualNum)) {pagelimit = actualNum;}
|
||||
}
|
||||
else {pagelimit = 1;}
|
||||
// 构造网页数据的对应数组
|
||||
const responseList = [];
|
||||
// 将第一页的数据加入数组
|
||||
responseList.push(response);
|
||||
// 创建不含第一页链接的数组
|
||||
const Links = [];
|
||||
for (let i = 1; i < pagelimit; i++) {
|
||||
Links.push(`${userUrl}&page=${i + 1}`);
|
||||
}
|
||||
// 由数组索引添加response
|
||||
await Promise.all(
|
||||
Links.map(async (link) => {
|
||||
const response = await got(link);
|
||||
responseList.push(response);
|
||||
})
|
||||
);
|
||||
// 由数组索引获取初步分析数据
|
||||
const tempData = [];
|
||||
await Promise.all(
|
||||
responseList.map(async (response) => {
|
||||
const result = await getData(response);
|
||||
tempData.push(result);
|
||||
})
|
||||
);
|
||||
// 拼接多页面item
|
||||
let realItem = [];
|
||||
for (const data of tempData) {
|
||||
realItem = realItem.concat(data.items);
|
||||
}
|
||||
// 构造最终data
|
||||
const data = {
|
||||
title:tempData[0].title,
|
||||
item:realItem,
|
||||
};
|
||||
// 生成RSS源码
|
||||
ctx.state.data = {
|
||||
title: `${title} - ${sortRename[sort]}`,
|
||||
title: `${data.title} - ${sortRename[sort]}(Page 1-${pagelimit})`,
|
||||
link: userUrl,
|
||||
item: items,
|
||||
item: data.item,
|
||||
};
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module.exports = {
|
||||
'/authors/:authorid/:sort?': ['xueli_sherryli'],
|
||||
'/characters/:characterid/:sort?': ['xueli_sherryli'],
|
||||
'/origins/:originid/:sort?': ['xueli_sherryli'],
|
||||
'/search/:keyword/:sort?': ['xueli_sherryli'],
|
||||
'/tags/:tagid/:sort?': ['xueli_sherryli'],
|
||||
'/authors/:authorid/:sort/:pagelimit?': ['xueli_sherryli'],
|
||||
'/characters/:characterid/:sort/:pagelimit?': ['xueli_sherryli'],
|
||||
'/origins/:originid/:sort/:pagelimit?': ['xueli_sherryli'],
|
||||
'/search/:keyword/:sort/:pagelimit?': ['xueli_sherryli'],
|
||||
'/tags/:tagid/:sort/:pagelimit?': ['xueli_sherryli'],
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module.exports = function (router) {
|
||||
router.get('/authors/:authorid/:sort?', require('./main'));
|
||||
router.get('/characters/:characterid/:sort?', require('./main'));
|
||||
router.get('/origins/:originid/:sort?', require('./main'));
|
||||
router.get('/search/:keyword/:sort?', require('./main'));
|
||||
router.get('/tags/:tagid/:sort?', require('./main'));
|
||||
router.get('/authors/:authorid/:sort/:pagelimit?', require('./main'));
|
||||
router.get('/characters/:characterid/:sort/:pagelimit?', require('./main'));
|
||||
router.get('/origins/:originid/:sort/:pagelimit?', require('./main'));
|
||||
router.get('/search/:keyword/:sort/:pagelimit?', require('./main'));
|
||||
router.get('/tags/:tagid/:sort/:pagelimit?', require('./main'));
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in New Issue