feat(route/pixiv): add R18 novels support and full content toggle for… (#17391)

* feat(route/pixiv): add R18 novels support and full content toggle for user novels

* fix: information & image placeholders

* refactor: split novels fetching into SFW/NSFW modules and improve type definitions

* feat: add info for sfw

* feat: add radar

* refactor: use jsdom instead of regex

* feat: add limit support for nsfw novels

* docs: rename radar title

* revert: part of #17440
Object.entries(options.searchParams) returns `[]`

* fix: clean up

* feat: early exit when no SFW novels found

* refactor: combine novel parsing logic into utils

* docs: restore pixiv doc link

* feat: cache novel content

* refactor: cleanup

* refactor: full content function

---------
This commit is contained in:
Tsuyumi 2024-11-06 03:56:50 +08:00 committed by GitHub
parent 6451863ee9
commit cf7ce2470b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 655 additions and 46 deletions

View File

@ -0,0 +1,22 @@
import got from '../pixiv-got';
import { maskHeader } from '../constants';
import queryString from 'query-string';
/**
*
* @param {string} illust_id id
* @param {string} token pixiv oauth token
* @returns {Promise<got.AxiosResponse<{illust: IllustDetail}>>}
*/
export default function getIllustDetail(illust_id: string, token: string) {
return got('https://app-api.pixiv.net/v1/illust/detail', {
headers: {
...maskHeader,
Authorization: 'Bearer ' + token,
},
searchParams: queryString.stringify({
illust_id,
filter: 'for_ios',
}),
});
}

View File

@ -0,0 +1,247 @@
import got from '../pixiv-got';
import { maskHeader } from '../constants';
import queryString from 'query-string';
import { config } from '@/config';
import { JSDOM, VirtualConsole } from 'jsdom';
import pixivUtils from '../utils';
import ConfigNotFoundError from '@/errors/types/config-not-found';
import cache from '@/utils/cache';
import { parseDate } from 'tough-cookie';
import { getToken } from '../token';
interface nsfwNovelWork {
id: string;
title: string;
caption: string;
restrict: number;
x_restrict: number;
is_original: boolean;
image_urls: {
square_medium: string;
medium: string;
large: string;
};
create_date: string;
tags: Array<{
name: string;
translated_name: string | null;
added_by_uploaded_user: boolean;
}>;
page_count: number;
text_length: number;
user: {
id: number;
name: string;
account: string;
profile_image_urls: {
medium: string;
};
is_followed: boolean;
is_access_blocking_user: boolean;
};
series?: {
id?: number;
title?: string;
};
total_bookmarks: number;
total_view: number;
total_comments: number;
}
interface nsfwNovelsResponse {
data: {
user: {
id: number;
name: string;
account: string;
profile_image_urls: {
medium: string;
};
is_followed: boolean;
is_access_blocking_user: boolean;
};
novels: nsfwNovelWork[];
};
}
interface nsfwNovelDetail {
id: string;
title: string;
seriesId: string | null;
seriesTitle: string | null;
seriesIsWatched: boolean | null;
userId: string;
coverUrl: string;
tags: string[];
caption: string;
cdate: string;
rating: {
like: number;
bookmark: number;
view: number;
};
text: string;
marker: null;
illusts: string[];
images: {
[key: string]: {
novelImageId: string;
sl: string;
urls: {
'240mw': string;
'480mw': string;
'1200x1200': string;
'128x128': string;
original: string;
};
};
};
seriesNavigation: {
nextNovel: null;
prevNovel: {
id: number;
viewable: boolean;
contentOrder: string;
title: string;
coverUrl: string;
viewableMessage: null;
} | null;
} | null;
glossaryItems: string[];
replaceableItemIds: string[];
aiType: number;
isOriginal: boolean;
}
function getNovels(user_id: string, token: string): Promise<nsfwNovelsResponse> {
return got('https://app-api.pixiv.net/v1/user/novels', {
headers: {
...maskHeader,
Authorization: 'Bearer ' + token,
},
searchParams: queryString.stringify({
user_id,
filter: 'for_ios',
}),
});
}
async function getNovelFullContent(novel_id: string, token: string): Promise<nsfwNovelDetail> {
return (await cache.tryGet(`https://app-api.pixiv.net/webview/v2/novel:${novel_id}`, async () => {
// https://github.com/mikf/gallery-dl/blob/main/gallery_dl/extractor/pixiv.py
// https://github.com/mikf/gallery-dl/commit/db507e30c7431d4ed7e23c153a044ce1751c2847
const response = await got('https://app-api.pixiv.net/webview/v2/novel', {
headers: {
...maskHeader,
Authorization: 'Bearer ' + token,
},
searchParams: queryString.stringify({
id: novel_id,
viewer_version: '20221031_ai',
}),
});
const virtualConsole = new VirtualConsole().on('error', () => void 0);
const { window } = new JSDOM(response.data, {
runScripts: 'dangerously',
virtualConsole,
});
const novelDetail = window.pixiv?.novel as nsfwNovelDetail;
window.close();
if (!novelDetail) {
throw new Error('No novel data found');
}
return novelDetail;
})) as nsfwNovelDetail;
}
function convertPixivProtocolExtended(caption: string): string {
const protocolMap = new Map([
[/pixiv:\/\/novels\/(\d+)/g, 'https://www.pixiv.net/novel/show.php?id=$1'],
[/pixiv:\/\/illusts\/(\d+)/g, 'https://www.pixiv.net/artworks/$1'],
[/pixiv:\/\/users\/(\d+)/g, 'https://www.pixiv.net/users/$1'],
[/pixiv:\/\/novel\/series\/(\d+)/g, 'https://www.pixiv.net/novel/series/$1'],
]);
let convertedText = caption;
for (const [pattern, replacement] of protocolMap) {
convertedText = convertedText.replace(pattern, replacement);
}
return convertedText;
}
export async function getR18Novels(id: string, fullContent: boolean, limit: number = 100) {
if (!config.pixiv || !config.pixiv.refreshToken) {
throw new ConfigNotFoundError(
'該用戶爲 R18 創作者,需要 PIXIV_REFRESHTOKEN。This user is an R18 creator, PIXIV_REFRESHTOKEN is required - pixiv RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>'
);
}
const token = await getToken(cache.tryGet);
if (!token) {
throw new ConfigNotFoundError('pixiv not login');
}
const response = await getNovels(id, token);
const novels = limit ? response.data.novels.slice(0, limit) : response.data.novels;
const username = novels[0].user.name;
const items = await Promise.all(
novels.map(async (novel) => {
const baseItem = {
title: novel.series?.title ? `${novel.series.title} - ${novel.title}` : novel.title,
description: `
<img src="${pixivUtils.getProxiedImageUrl(novel.image_urls.large)}" />
<p>${convertPixivProtocolExtended(novel.caption) || ''}</p>
<p>
${novel.text_length}<br>
${novel.total_view}<br>
${novel.total_bookmarks}<br>
${novel.total_comments}<br>
</p>`,
author: novel.user.name,
pubDate: parseDate(novel.create_date),
link: `https://www.pixiv.net/novel/show.php?id=${novel.id}`,
category: novel.tags.map((t) => t.name),
};
if (!fullContent) {
return baseItem;
}
try {
const novelDetail = await getNovelFullContent(novel.id, token);
const images = Object.fromEntries(
Object.entries(novelDetail.images)
.filter(([, image]) => image?.urls?.original)
.map(([id, image]) => [id, image.urls.original.replace('https://i.pximg.net', config.pixiv.imgProxy || '')])
);
const content = await pixivUtils.parseNovelContent(novelDetail.text, images, token);
return {
...baseItem,
description: `${baseItem.description}<hr>${content}`,
};
} catch {
return baseItem;
}
})
);
return {
title: `${username}'s novels - pixiv`,
description: `${username} 的 pixiv 最新小说`,
image: pixivUtils.getProxiedImageUrl(novels[0].user.profile_image_urls.medium),
link: `https://www.pixiv.net/users/${id}/novels`,
item: items,
};
}

View File

@ -0,0 +1,201 @@
import got from '@/utils/got';
import cache from '@/utils/cache';
import pixivUtils from '../utils';
import { parseDate } from '@/utils/parse-date';
const baseUrl = 'https://www.pixiv.net';
interface sfwNovelWork {
id: string;
title: string;
genre: string;
xRestrict: number;
restrict: number;
url: string;
tags: string[];
userId: string;
userName: string;
profileImageUrl: string;
textCount: number;
wordCount: number;
readingTime: number;
useWordCount: boolean;
description: string;
isBookmarkable: boolean;
bookmarkData: null;
bookmarkCount: number;
isOriginal: boolean;
marker: null;
titleCaptionTranslation: {
workTitle: null;
workCaption: null;
};
createDate: string;
updateDate: string;
isMasked: boolean;
aiType: number;
seriesId: string;
seriesTitle: string;
isUnlisted: boolean;
}
interface sfwNovelsResponse {
data: {
error: boolean;
message: string;
body: {
works: Record<string, sfwNovelWork>;
extraData: {
meta: {
title: string;
description: string;
canonical: string;
ogp: {
description: string;
image: string;
title: string;
type: string;
};
twitter: {
description: string;
image: string;
title: string;
card: string;
};
alternateLanguages: {
ja: string;
en: string;
};
descriptionHeader: string;
};
};
};
};
}
interface sfwNovelDetail {
body: {
content: string;
textEmbeddedImages: Record<
string,
{
novelImageId: string;
sl: string;
urls: {
original: string;
'1200x1200': string;
'480mw': string;
'240mw': string;
'128x128': string;
};
}
>;
};
}
async function getNovelFullContent(novel_id: string): Promise<{ content: string; images: Record<string, string> }> {
const url = `${baseUrl}/ajax/novel/${novel_id}`;
return (await cache.tryGet(url, async () => {
const response = await got(url, {
headers: {
referer: `${baseUrl}/novel/show.php?id=${novel_id}`,
},
});
const novelDetail = response.data as sfwNovelDetail;
if (!novelDetail) {
throw new Error('No novel data found');
}
const images: Record<string, string> = {};
if (novelDetail.body.textEmbeddedImages) {
for (const [id, image] of Object.entries(novelDetail.body.textEmbeddedImages)) {
images[id] = pixivUtils.getProxiedImageUrl(image.urls.original);
}
}
return {
content: novelDetail.body.content,
images,
};
})) as { content: string; images: Record<string, string> };
}
export async function getNonR18Novels(id: string, fullContent: boolean, limit: number = 100) {
const url = `${baseUrl}/users/${id}/novels`;
const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
headers: {
referer: url,
},
});
const novels = Object.keys(allData.body.novels)
.sort((a, b) => Number(b) - Number(a))
.slice(0, Number.parseInt(String(limit), 10));
if (novels.length === 0) {
throw new Error('No novels found, fallback to R18 API');
// Throw error early to avoid unnecessary API requests
// Since hasPixivAuth() check failed earlier and R18 API requires authentication, this will result in ConfigNotFoundError
}
const searchParams = new URLSearchParams();
for (const novel of novels) {
searchParams.append('ids[]', novel);
}
const { data } = (await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
headers: {
referer: url,
},
searchParams,
})) as sfwNovelsResponse;
const items = await Promise.all(
Object.values(data.body.works).map(async (item) => {
const baseItem = {
title: item.title,
description: `
<img src=${pixivUtils.getProxiedImageUrl(item.url)} />
<p>${item.description}</p>
<p>
${item.textCount}<br>
${item.readingTime} <br>
${item.bookmarkCount}<br>
</p>
`,
link: `${baseUrl}/novel/show.php?id=${item.id}`,
author: item.userName,
pubDate: parseDate(item.createDate),
updated: parseDate(item.updateDate),
category: item.tags,
};
if (!fullContent) {
return baseItem;
}
try {
const { content: initialContent, images } = await getNovelFullContent(item.id);
const content = await pixivUtils.parseNovelContent(initialContent, images);
return {
...baseItem,
description: `${baseItem.description}<hr>${content}`,
};
} catch {
return baseItem;
}
})
);
return {
title: data.body.extraData.meta.title,
description: data.body.extraData.meta.ogp.description,
image: pixivUtils.getProxiedImageUrl(Object.values(data.body.works)[0].profileImageUrl),
link: url,
item: items,
};
}

View File

@ -1,15 +1,36 @@
import { Route } from '@/types';
import got from '@/utils/got';
import { parseDate } from '@/utils/parse-date';
const baseUrl = 'https://www.pixiv.net';
import { Data, Route, ViewType } from '@/types';
import { fallback, queryToBoolean } from '@/utils/readable-social';
import { getR18Novels } from './api/get-novels-nsfw';
import { getNonR18Novels } from './api/get-novels-sfw';
import { config } from '@/config';
export const route: Route = {
path: '/user/novels/:id',
path: '/user/novels/:id/:full_content?',
categories: ['social-media'],
view: ViewType.Articles,
example: '/pixiv/user/novels/27104704',
parameters: { id: "User id, available in user's homepage URL" },
parameters: {
id: "User id, available in user's homepage URL",
full_content: {
description: 'Enable or disable the display of full content. ',
options: [
{ value: 'true', label: 'true' },
{ value: 'false', label: 'false' },
],
default: 'false',
},
},
features: {
requireConfig: false,
requireConfig: [
{
name: 'PIXIV_REFRESHTOKEN',
optional: true,
description: `
Pixiv refresh_token R18
refresh_token after Pixiv login, required for accessing R18 novels
[https://docs.rsshub.app/deploy/config#pixiv](https://docs.rsshub.app/deploy/config#pixiv)`,
},
],
requirePuppeteer: false,
antiCrawler: false,
supportBT: false,
@ -18,54 +39,53 @@ export const route: Route = {
},
radar: [
{
title: 'User Novels (簡介 Basic info)',
source: ['www.pixiv.net/users/:id/novels'],
target: '/user/novels/:id',
},
{
title: 'User Novels (全文 Full text)',
source: ['www.pixiv.net/users/:id/novels'],
target: '/user/novels/:id/true',
},
],
name: 'User Novels',
maintainers: ['TonyRL'],
maintainers: ['TonyRL', 'SnowAgar25'],
handler,
description: `
| Novel Type | full_content | PIXIV_REFRESHTOKEN | Content |
|-------------------|--------------|-------------------|-----------------|
| Non R18 | false | Not Required | Basic info |
| Non R18 | true | Not Required | Full text |
| R18 | false | Required | Basic info |
| R18 | true | Required | Full text |
Default value for \`full_content\` is \`false\` if not specified.
Example:
- \`/pixiv/user/novels/79603797\` → 簡介 Basic info
- \`/pixiv/user/novels/79603797/true\` → 全文 Full text`,
};
async function handler(ctx) {
const id = ctx.req.param('id');
const { limit = 100 } = ctx.req.query();
const url = `${baseUrl}/users/${id}/novels`;
const { data: allData } = await got(`${baseUrl}/ajax/user/${id}/profile/all`, {
headers: {
referer: url,
},
});
const hasPixivAuth = () => Boolean(config.pixiv && config.pixiv.refreshToken);
const novels = Object.keys(allData.body.novels)
.sort((a, b) => b - a)
.slice(0, Number.parseInt(limit, 10));
const searchParams = new URLSearchParams();
for (const novel of novels) {
searchParams.append('ids[]', novel);
async function handler(ctx): Promise<Data> {
const id = ctx.req.param('id');
const fullContent = fallback(undefined, queryToBoolean(ctx.req.param('full_content')), false);
const { limit } = ctx.req.query();
// Use R18 API first if auth exists
if (hasPixivAuth()) {
return await getR18Novels(id, fullContent, limit);
}
const { data } = await got(`${baseUrl}/ajax/user/${id}/profile/novels`, {
headers: {
referer: url,
},
searchParams,
});
// Attempt non-R18 API when Pixiv auth is missing
const nonR18Result = await getNonR18Novels(id, fullContent, limit).catch(() => null);
if (nonR18Result) {
return nonR18Result;
}
const items = Object.values(data.body.works).map((item) => ({
title: item.seriesTitle || item.title,
description: item.description || item.title,
link: `${baseUrl}/novel/series/${item.id}`,
author: item.userName,
pubDate: parseDate(item.createDate),
updated: parseDate(item.updateDate),
category: item.tags,
}));
return {
title: data.body.extraData.meta.title,
description: data.body.extraData.meta.ogp.description,
image: Object.values(data.body.works)[0].profileImageUrl,
link: url,
item: items,
};
// Fallback to R18 API as last resort
return await getR18Novels(id, fullContent, limit);
}

View File

@ -1,4 +1,6 @@
import { config } from '@/config';
import { load } from 'cheerio';
import getIllustDetail from './api/get-illust-detail';
export default {
getImgs(illust) {
@ -14,4 +16,121 @@ export default {
}
return images;
},
getProxiedImageUrl(originalUrl: string): string {
return originalUrl.replace('https://i.pximg.net', config.pixiv.imgProxy || '');
},
// docs: https://www.pixiv.help/hc/ja/articles/235584168-小説作品の本文内に使える特殊タグとは
async parseNovelContent(content: string, images: Record<string, string>, token?: string): Promise<string> {
try {
// 如果有 token處理 pixiv 圖片引用
// If token exists, process pixiv image references
if (token) {
const imageMatches = [...content.matchAll(/\[pixivimage:(\d+)(?:-(\d+))?\]/g)];
const imageIdToUrl = new Map<string, string>();
// 批量獲取圖片資訊
// Batch fetch image information
await Promise.all(
imageMatches.map(async ([, illustId, pageNum]) => {
if (!illustId) {
return;
}
try {
const illust = (await getIllustDetail(illustId, token)).data.illust;
const pixivimages = this.getImgs(illust).map((img) => img.match(/src="([^"]+)"/)?.[1] || '');
const imageUrl = pixivimages[Number(pageNum) || 0];
if (imageUrl) {
imageIdToUrl.set(pageNum ? `${illustId}-${pageNum}` : illustId, imageUrl);
}
} catch (error) {
// 記錄錯誤但不中斷處理
// Log error but don't interrupt processing
logger.warn(`Failed to fetch illust detail for ID ${illustId}: ${error instanceof Error ? error.message : String(error)}`);
}
})
);
// 替換 pixiv 圖片引用為 img 標籤
// Replace pixiv image references with img tags
content = content.replaceAll(/\[pixivimage:(\d+)(?:-(\d+))?\]/g, (match, illustId, pageNum) => {
const key = pageNum ? `${illustId}-${pageNum}` : illustId;
const imageUrl = imageIdToUrl.get(key);
return imageUrl ? `<img src="${imageUrl}" alt="pixiv illustration ${illustId}${pageNum ? ` page ${pageNum}` : ''}">` : match;
});
} else {
/*
* get-novels-sfw
* PIXIV_REFRESHTOKEN [pixivimage:(\d+)] artwork
* Pixiv
*
* Handle get-novels-sfw case
* When PIXIV_REFRESHTOKEN is not available, convert [pixivimage:(\d+)] format to artwork link
* Provide direct link to original artwork page since artwork details cannot be retrieved
*/
content = content.replaceAll(/\[pixivimage:(\d+)(?:-(\d+))?\]/g, (_, illustId) => `<a href="https://www.pixiv.net/artworks/${illustId}" target="_blank" rel="noopener noreferrer">Pixiv Artwork #${illustId}</a>`);
}
// 處理作者上傳的圖片
// Process author uploaded images
content = content.replaceAll(/\[uploadedimage:(\d+)\]/g, (match, imageId) => {
if (images[imageId]) {
return `<img src="${images[imageId]}" alt="novel illustration ${imageId}">`;
}
return match;
});
// 基本格式處理
// Basic formatting
content = content
// 換行轉換為 HTML 換行
// Convert newlines to HTML breaks
.replaceAll('\n', '<br>')
// 連續換行轉換為段落
// Convert consecutive breaks to paragraphs
.replaceAll(/(<br>){2,}/g, '</p><p>')
// ruby 標籤(為日文漢字標註讀音)
// ruby tags (for Japanese kanji readings)
.replaceAll(/\[\[rb:(.*?)>(.*?)\]\]/g, '<ruby>$1<rt>$2</rt></ruby>')
// 外部連結
// external links
.replaceAll(/\[\[jumpuri:(.*?)>(.*?)\]\]/g, '<a href="$2" target="_blank" rel="noopener noreferrer">$1</a>')
// 頁面跳轉,但由於 [newpage] 使用 hr 分隔,沒有頁數,沒必要跳轉,所以只顯示文字
// Page jumps, but since [newpage] uses hr separators, without the page numbers, jumping isn't needed, so just display text
.replaceAll(/\[jump:(\d+)\]/g, 'Jump to page $1')
// 章節標題
// chapter titles
.replaceAll(/\[chapter:(.*?)\]/g, '<h2>$1</h2>')
// 分頁符
// page breaks
.replaceAll('[newpage]', '<hr>');
// 使用 cheerio 進行 HTML 清理和優化
// Use cheerio for HTML cleanup and optimization
const $content = load(`<article><p>${content}</p></article>`);
// 處理嵌套段落:移除多餘的嵌套
// Handle nested paragraphs: remove unnecessary nesting
$content('p p').each((_, elem) => {
const $elem = $content(elem);
$elem.replaceWith($elem.html() || '');
});
// 處理段落中的標題:確保正確的 HTML 結構
// Handle headings in paragraphs: ensure correct HTML structure
$content('p h2').each((_, elem) => {
const $elem = $content(elem);
const $parent = $elem.parent('p');
const html = $elem.prop('outerHTML');
if ($parent.length && html) {
$parent.replaceWith(`</p>${html}<p>`);
}
});
return $content.html() || '';
} catch (error) {
throw new Error(`Error parsing novel content: ${error instanceof Error ? error.message : String(error)}`);
}
},
};