feat(route/locals): add content feed (#21542)
* feat(route/locals): add authenticated feed route Add a Locals route with auth_token-based feed fetching, classic-to-beta URL fallback, and support for community, home, content, and content_plus feed views. Parse feed pages into RSS items using shared post selectors while surfacing clear errors when access is unavailable. * feat(route/locals): add content feed Add a Locals content route that uses the authenticated session-backed server functions instead of brittle page scraping. Support tier filters and media-type filters so paid and content-library posts can be subscribed to reliably. * fix(route/locals): include content thumbnails Use Locals preview and audio image fields so content and podcast entries expose thumbnails consistently. Render the image into item descriptions and standard image fields for better feed reader compatibility. * fix(route/locals): include regular content posts Keep Locals content feeds from dropping regular text and photo posts that arrive as no_content entries with real body content. Preserve image handling while leaving plus feeds unchanged when the source only returns five content-plus items. * fix(route/locals): prefer content poster images Prefer Locals poster-style preview images over raw first-frame captures when both are available. Keep the raw frame fallback for posts where the API does not expose a styled poster image. * fix(route/locals): resolve runtime action ids Derive the Locals feed action id from the current runtime assets instead of hardcoding build-specific server function ids. Tighten option validation with own-key checks and align the radar source with supported matching rules. * fix(config): sort locals session settings Move the Locals session env key and config blocks into alphabetical order so the route-specific configuration matches the repository's config layout rules. * fix(config): reorder locals session settings * fix(route/locals): simplify route and request handling * fix(config): restore locals config ordering * fix(config): sort security key entry * fix(route/locals): adjust message clarity of path parameter necessity
This commit is contained in:
Luke D Williams
GitHub
parent
6ae3f644bf
commit
482ad1cb80
|
|
@ -131,7 +131,7 @@ type ConfigEnvKeys =
|
|||
| 'JUMEILI_COOKIE'
|
||||
| 'KEYLOL_COOKIE'
|
||||
| 'LASTFM_API_KEY'
|
||||
| 'SECURITY_KEY'
|
||||
| 'LOCALS_SESSION'
|
||||
| 'LOFTER_COOKIE'
|
||||
| 'LORIENTLEJOUR_TOKEN'
|
||||
| 'LORIENTLEJOUR_USERNAME'
|
||||
|
|
@ -182,6 +182,7 @@ type ConfigEnvKeys =
|
|||
| 'SCIHUB_HOST'
|
||||
| 'SDO_FF14RISINGSTONES'
|
||||
| 'SDO_UA'
|
||||
| 'SECURITY_KEY'
|
||||
| 'SIS001_BASE_URL'
|
||||
| 'SKEB_BEARER_TOKEN'
|
||||
| 'SORRYCC_COOKIES'
|
||||
|
|
@ -470,6 +471,9 @@ export type Config = {
|
|||
lightnovel: {
|
||||
cookie?: string;
|
||||
};
|
||||
locals: {
|
||||
session?: string;
|
||||
};
|
||||
lofter: {
|
||||
cookies?: string;
|
||||
};
|
||||
|
|
@ -959,6 +963,9 @@ const calculateValue = () => {
|
|||
lightnovel: {
|
||||
cookie: envs.SECURITY_KEY,
|
||||
},
|
||||
locals: {
|
||||
session: envs.LOCALS_SESSION,
|
||||
},
|
||||
lofter: {
|
||||
cookies: envs.LOFTER_COOKIE,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -0,0 +1,436 @@
|
|||
import vm from 'node:vm';
|
||||
|
||||
import { config } from '@/config';
|
||||
import ConfigNotFoundError from '@/errors/types/config-not-found';
|
||||
import type { DataItem, Route } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
|
||||
const contentFilterMap = {
|
||||
nonplus: 'regular_content',
|
||||
plus: 'content_plus',
|
||||
} as const;
|
||||
|
||||
const contentTypeMap = {
|
||||
article: 'article',
|
||||
audio: 'audio',
|
||||
document: 'document',
|
||||
live: 'live_stream',
|
||||
live_stream: 'live_stream',
|
||||
pdf: 'document',
|
||||
podcast: 'audio',
|
||||
podcasts: 'audio',
|
||||
video: 'video',
|
||||
} as const;
|
||||
|
||||
const rootUrl = 'https://locals.com';
|
||||
|
||||
type ContentFilter = keyof typeof contentFilterMap;
|
||||
type ContentType = keyof typeof contentTypeMap;
|
||||
|
||||
type LocalsPost = {
|
||||
audios?: Array<{
|
||||
preview?: string;
|
||||
}>;
|
||||
author_name?: string;
|
||||
author_username?: string;
|
||||
content_plus?: {
|
||||
enabled?: boolean;
|
||||
};
|
||||
content_type?: string;
|
||||
photos?: Array<{
|
||||
sizes?: {
|
||||
full?: {
|
||||
url?: string;
|
||||
};
|
||||
thumb?: {
|
||||
url?: string;
|
||||
};
|
||||
};
|
||||
}>;
|
||||
post_date?: string;
|
||||
published?: string;
|
||||
previews?: Array<{
|
||||
first_frame_url?: string;
|
||||
url?: string;
|
||||
}>;
|
||||
share_url?: string;
|
||||
subtitle?: string;
|
||||
text?: string;
|
||||
title?: string;
|
||||
is_content?: boolean;
|
||||
videos?: Array<{
|
||||
preview?: string;
|
||||
source?: string;
|
||||
}>;
|
||||
};
|
||||
|
||||
type LocalsResponse = {
|
||||
code: string;
|
||||
data: LocalsPost[];
|
||||
};
|
||||
|
||||
type LocalsCommunityInfo = {
|
||||
description?: string;
|
||||
design?: {
|
||||
image?: {
|
||||
big?: string;
|
||||
thumb?: string;
|
||||
};
|
||||
};
|
||||
hashtag: string;
|
||||
id: number;
|
||||
title: string;
|
||||
};
|
||||
|
||||
export const route: Route = {
|
||||
path: '/content/:community/:option1?/:option2?',
|
||||
categories: ['social-media'],
|
||||
example: '/locals/content/mattfradd/video',
|
||||
parameters: {
|
||||
community: 'Community slug from `locals.com/:community/feed?mode=content`',
|
||||
option1: 'Filter or content type. Filters: `plus`, `nonplus`. Content types: `video`, `live`, `audio`, `podcast`, `article`, `document`, `pdf`',
|
||||
option2: 'Content type when `option1` is a filter',
|
||||
},
|
||||
description: 'Fetches the Locals content library with an authenticated session cookie. By default it merges regular content and content+ posts, and it can be filtered by access tier and media type.',
|
||||
features: {
|
||||
requireConfig: [
|
||||
{
|
||||
name: 'LOCALS_SESSION',
|
||||
description: 'The value of the `locals2.v3.session` cookie after logging in to Locals',
|
||||
},
|
||||
],
|
||||
requirePuppeteer: false,
|
||||
antiCrawler: true,
|
||||
supportBT: false,
|
||||
supportPodcast: false,
|
||||
supportScihub: false,
|
||||
},
|
||||
radar: [
|
||||
{
|
||||
source: ['locals.com/:community/feed'],
|
||||
target: '/content/:community',
|
||||
},
|
||||
],
|
||||
name: 'Content Feed',
|
||||
maintainers: ['luckycold'],
|
||||
handler,
|
||||
};
|
||||
|
||||
function getCookieHeader(session: string) {
|
||||
return [`locals2.v3.session=${session}`, 'locals.preferLocals2=false', 'locals2.v3.locale.actual=en-us%2Cen', 'locals2.v3.locale.inferred=en'].join('; ');
|
||||
}
|
||||
|
||||
function getRequestHeaders(session: string) {
|
||||
return {
|
||||
Cookie: getCookieHeader(session),
|
||||
};
|
||||
}
|
||||
|
||||
function extractAssetUrls(html: string) {
|
||||
return [...new Set([...html.matchAll(/\/_build\/assets\/[^"'\s>]+\.js/g)].map((match) => match[0]))];
|
||||
}
|
||||
|
||||
function extractFeedActionId(asset: string) {
|
||||
const symbol = asset.match(/getFeedPosts:([A-Za-z_$][\w$]*)/)?.[1];
|
||||
if (!symbol) {
|
||||
return;
|
||||
}
|
||||
|
||||
const match = asset.match(new RegExp(`${symbol}=s\\(\\(\\)=>\\{\\},"([^"]+)","([^"]+)"\\)`));
|
||||
if (!match) {
|
||||
return;
|
||||
}
|
||||
|
||||
return `${match[1]}#${match[2]}`;
|
||||
}
|
||||
|
||||
function createRequestBody(value: unknown) {
|
||||
let nextIndex = 0;
|
||||
|
||||
const encode = (input: unknown): object => {
|
||||
if (typeof input === 'number') {
|
||||
return { t: 0, s: input };
|
||||
}
|
||||
|
||||
if (typeof input === 'string') {
|
||||
return { t: 1, s: input };
|
||||
}
|
||||
|
||||
if (Array.isArray(input)) {
|
||||
const i = nextIndex++;
|
||||
return {
|
||||
a: input.map((item) => encode(item)),
|
||||
i,
|
||||
l: input.length,
|
||||
o: 0,
|
||||
t: 9,
|
||||
};
|
||||
}
|
||||
|
||||
if (input && typeof input === 'object') {
|
||||
const i = nextIndex++;
|
||||
const entries = Object.entries(input).filter(([, item]) => item !== undefined);
|
||||
|
||||
return {
|
||||
i,
|
||||
o: 0,
|
||||
p: {
|
||||
k: entries.map(([key]) => key),
|
||||
s: entries.length,
|
||||
v: entries.map(([, item]) => encode(item)),
|
||||
},
|
||||
t: 10,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error('Unsupported Locals request payload.');
|
||||
};
|
||||
|
||||
return JSON.stringify({
|
||||
f: 31,
|
||||
m: [],
|
||||
t: encode(value),
|
||||
});
|
||||
}
|
||||
|
||||
function parseUnknownResponse<T>(body: string, instanceId: string): T {
|
||||
const sandbox = {
|
||||
$R: {},
|
||||
self: {},
|
||||
} as {
|
||||
$R: Record<string, unknown[]>;
|
||||
self: {
|
||||
$R?: Record<string, unknown[]>;
|
||||
};
|
||||
};
|
||||
|
||||
sandbox.self.$R = sandbox.$R;
|
||||
|
||||
vm.createContext(sandbox);
|
||||
vm.runInContext(body, sandbox, { timeout: 5000 });
|
||||
|
||||
const value = sandbox.self.$R?.[instanceId]?.[0] as T | undefined;
|
||||
if (!value) {
|
||||
throw new Error('Unable to decode the Locals server response.');
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function extractCommunityInfo(html: string, community: string) {
|
||||
const markerIndex = html.indexOf(`hashtag:"${community}"`);
|
||||
if (markerIndex === -1) {
|
||||
throw new Error('Unable to resolve the Locals community metadata.');
|
||||
}
|
||||
|
||||
const context = html.slice(Math.max(0, markerIndex - 200), markerIndex + 1200);
|
||||
const idMatch = context.match(/id:(\d+)/);
|
||||
const titleMatch = context.match(/title:"([^"]+)"/);
|
||||
const descriptionMatch = context.match(/description:"([^"]*)"/);
|
||||
const imageMatch = context.match(/image:\$R\[\d+\]=\{big:"([^"]*)",thumb:"([^"]*)"/);
|
||||
|
||||
if (!idMatch || !titleMatch) {
|
||||
throw new Error('Unable to resolve the Locals community metadata.');
|
||||
}
|
||||
|
||||
return {
|
||||
description: descriptionMatch?.[1],
|
||||
design: {
|
||||
image: {
|
||||
big: imageMatch?.[1] || undefined,
|
||||
thumb: imageMatch?.[2] || undefined,
|
||||
},
|
||||
},
|
||||
hashtag: community,
|
||||
id: Number(idMatch[1]),
|
||||
title: titleMatch[1],
|
||||
} satisfies LocalsCommunityInfo;
|
||||
}
|
||||
|
||||
function fetchContentPage(community: string, session: string) {
|
||||
return cache.tryGet(`locals:content-page:${community}`, () => ofetch(`${rootUrl}/${community}/feed?mode=content`, { headers: getRequestHeaders(session) }));
|
||||
}
|
||||
|
||||
function resolveActionIds(community: string, session: string) {
|
||||
return cache.tryGet(`locals:action-ids:${community}`, async () => {
|
||||
const html = await fetchContentPage(community, session);
|
||||
const serverApiAsset = extractAssetUrls(html).find((assetUrl) => assetUrl.includes('/serverApi-'));
|
||||
if (!serverApiAsset) {
|
||||
throw new Error('Unable to locate the current Locals serverApi asset.');
|
||||
}
|
||||
|
||||
const assetContent = await ofetch(new URL(serverApiAsset, rootUrl).href, {
|
||||
headers: getRequestHeaders(session),
|
||||
});
|
||||
const serverId = extractFeedActionId(assetContent);
|
||||
|
||||
if (!serverId) {
|
||||
throw new Error('Unable to discover the current Locals feed action id.');
|
||||
}
|
||||
|
||||
return serverId;
|
||||
});
|
||||
}
|
||||
|
||||
function getImage(post: LocalsPost) {
|
||||
return post.photos?.[0]?.sizes?.full?.url || post.photos?.[0]?.sizes?.thumb?.url || post.previews?.[0]?.url || post.previews?.[0]?.first_frame_url || post.videos?.[0]?.preview || post.audios?.[0]?.preview;
|
||||
}
|
||||
|
||||
function getTitle(post: LocalsPost) {
|
||||
const textFallback = post.text
|
||||
?.replaceAll(/<[^>]+>/g, '\n')
|
||||
.split('\n')
|
||||
.map((line) => line.trim())
|
||||
.find(Boolean);
|
||||
return post.title || post.subtitle || textFallback || post.share_url || 'Locals post';
|
||||
}
|
||||
|
||||
function renderDescription(image: string | undefined, description: string | undefined) {
|
||||
if (image && description) {
|
||||
return `<p><img src="${image}"></p>${description}`;
|
||||
}
|
||||
|
||||
if (image) {
|
||||
return `<p><img src="${image}"></p>`;
|
||||
}
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
function mapPostToItem(post: LocalsPost): DataItem | null {
|
||||
const image = getImage(post);
|
||||
const hasBodyContent = Boolean(post.text || image || post.photos?.length || post.videos?.length || post.audios?.length);
|
||||
|
||||
if (!post.share_url || (!post.is_content && post.content_type === 'no_content' && !hasBodyContent)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const contentType = post.content_type && post.content_type !== 'no_content' ? [post.content_type] : [];
|
||||
const accessType = post.content_plus?.enabled ? ['content_plus'] : ['content'];
|
||||
|
||||
return {
|
||||
author: post.author_name || post.author_username,
|
||||
category: [...accessType, ...contentType],
|
||||
description: renderDescription(image, post.text),
|
||||
image,
|
||||
itunes_item_image: image,
|
||||
link: post.share_url,
|
||||
pubDate: post.published ? parseDate(post.published) : post.post_date ? parseDate(post.post_date) : undefined,
|
||||
title: getTitle(post),
|
||||
};
|
||||
}
|
||||
|
||||
function parseOptions(option1: string | undefined, option2: string | undefined) {
|
||||
const values = [option1, option2].filter(Boolean) as string[];
|
||||
const hasFilter = (value: string): value is ContentFilter => Object.hasOwn(contentFilterMap, value);
|
||||
const hasContentType = (value: string): value is ContentType => Object.hasOwn(contentTypeMap, value);
|
||||
const filter = values.find((value) => hasFilter(value));
|
||||
const contentType = values.find((value) => hasContentType(value));
|
||||
const unknown = values.find((value) => !hasFilter(value) && !hasContentType(value));
|
||||
|
||||
if (unknown) {
|
||||
throw new Error('Invalid Locals content route option. Supported filters are `plus` and `nonplus`, and supported content types are `video`, `live`, `audio`, `podcast`, `article`, `document`, and `pdf`.');
|
||||
}
|
||||
|
||||
return {
|
||||
contentType: contentType ? contentTypeMap[contentType] : undefined,
|
||||
filter,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestServerFunction<T>(session: string, id: string, key: string, args: unknown[]) {
|
||||
const response = await ofetch.raw(`${rootUrl}/_server`, {
|
||||
body: createRequestBody(args),
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
...getRequestHeaders(session),
|
||||
'X-Server-Id': id,
|
||||
'X-Server-Instance': key,
|
||||
},
|
||||
method: 'POST',
|
||||
responseType: 'text',
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Unable to access the Locals server function (${response.status}).`);
|
||||
}
|
||||
|
||||
return parseUnknownResponse<T>(response._data, key);
|
||||
}
|
||||
|
||||
function fetchCommunityInfo(community: string, session: string) {
|
||||
return cache.tryGet(`locals:community:${community}`, async () => {
|
||||
const html = await fetchContentPage(community, session);
|
||||
return extractCommunityInfo(html, community);
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchFeedData(communityId: number, community: string, session: string, serverId: string, filter: ContentFilter | undefined, contentType: string | undefined) {
|
||||
const requestFilter = filter ? contentFilterMap[filter] : undefined;
|
||||
const filters = requestFilter ? [requestFilter] : Object.values(contentFilterMap);
|
||||
|
||||
const responses = await Promise.all(
|
||||
filters.map((currentFilter) =>
|
||||
cache.tryGet(`locals:data:${communityId}:${community}:${currentFilter}:${contentType ?? 'all'}`, () =>
|
||||
requestServerFunction<LocalsResponse>(session, serverId, `server-fn:rsshub-${community}-${currentFilter}-${contentType ?? 'all'}`, [
|
||||
{
|
||||
communityId,
|
||||
contentType,
|
||||
filter: currentFilter,
|
||||
order: 'recent',
|
||||
page: 1,
|
||||
pageSize: 20,
|
||||
},
|
||||
])
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
const items = new Map<string, DataItem>();
|
||||
|
||||
for (const response of responses) {
|
||||
for (const post of response.data) {
|
||||
const item = mapPostToItem(post);
|
||||
if (!item?.link) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!items.has(item.link)) {
|
||||
items.set(item.link, item);
|
||||
continue;
|
||||
}
|
||||
|
||||
const existing = items.get(item.link);
|
||||
if (existing) {
|
||||
existing.category = [...new Set([...(existing.category ?? []), ...(item.category ?? [])])];
|
||||
existing.description = existing.description || item.description;
|
||||
existing.itunes_item_image = existing.itunes_item_image || item.itunes_item_image;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...items.values()];
|
||||
}
|
||||
|
||||
async function handler(ctx) {
|
||||
const { community, option1, option2 } = ctx.req.param();
|
||||
|
||||
if (!config.locals?.session) {
|
||||
throw new ConfigNotFoundError('Locals RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
|
||||
}
|
||||
|
||||
const { contentType, filter } = parseOptions(option1, option2);
|
||||
const serverId = await resolveActionIds(community, config.locals.session);
|
||||
const communityInfo = await fetchCommunityInfo(community, config.locals.session);
|
||||
const items = await fetchFeedData(communityInfo.id, community, config.locals.session, serverId, filter, contentType);
|
||||
|
||||
return {
|
||||
description: `Locals content feed for ${communityInfo.title}${filter ? ` (${filter})` : ''}${contentType ? ` (${contentType})` : ''}`,
|
||||
image: communityInfo.design?.image?.big || communityInfo.design?.image?.thumb,
|
||||
item: items,
|
||||
link: `https://locals.com/${community}/feed?mode=content${contentType ? `&content=${contentType}` : ''}`,
|
||||
title: `Locals - ${communityInfo.title}`,
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
import type { Namespace } from '@/types';
|
||||
|
||||
export const namespace: Namespace = {
|
||||
name: 'Locals',
|
||||
url: 'locals.com',
|
||||
lang: 'en',
|
||||
};
|
||||
Loading…
Reference in New Issue