diff --git a/lib/routes/youtube/api/google.ts b/lib/routes/youtube/api/google.ts new file mode 100644 index 000000000..3ad8de903 --- /dev/null +++ b/lib/routes/youtube/api/google.ts @@ -0,0 +1,133 @@ +import { google } from 'googleapis'; +const { OAuth2 } = google.auth; +import { config } from '@/config'; +import utils, { getVideoUrl } from '../utils'; +import cache from '@/utils/cache'; +import { parseDate } from '@/utils/parse-date'; +import ofetch from '@/utils/ofetch'; +import * as cheerio from 'cheerio'; +import NotFoundError from '@/errors/types/not-found'; +import { Data } from '@/types'; + +let count = 0; +const youtube = {}; +if (config.youtube && config.youtube.key) { + const keys = config.youtube.key.split(','); + + for (const [index, key] of keys.entries()) { + if (key) { + youtube[index] = google.youtube({ + version: 'v3', + auth: key, + }); + count = index + 1; + } + } +} + +let index = -1; +const exec = async (func) => { + let result; + for (let i = 0; i < count; i++) { + index++; + try { + // eslint-disable-next-line no-await-in-loop + result = await func(youtube[index % count]); + break; + } catch { + // console.error(error); + } + } + return result; +}; + +let youtubeOAuth2Client; +if (config.youtube && config.youtube.clientId && config.youtube.clientSecret && config.youtube.refreshToken) { + youtubeOAuth2Client = new OAuth2(config.youtube.clientId, config.youtube.clientSecret, 'https://developers.google.com/oauthplayground'); + youtubeOAuth2Client.setCredentials({ refresh_token: config.youtube.refreshToken }); +} + +export { youtubeOAuth2Client, exec }; + +export const getDataByUsername = async ({ username, embed, filterShorts }: { username: string; embed: boolean; filterShorts: boolean }): Promise => { + let userHandleData; + if (username.startsWith('@')) { + userHandleData = await cache.tryGet(`youtube:handle:${username}`, async () => { + const link = `https://www.youtube.com/${username}`; + const response = await ofetch(link); + const $ = cheerio.load(response); + const ytInitialData = JSON.parse( + $('script') + .text() + .match(/ytInitialData = ({.*?});/)?.[1] || '{}' + ); + const metadataRenderer = ytInitialData.metadata.channelMetadataRenderer; + + const channelId = metadataRenderer.externalId; + const channelName = metadataRenderer.title; + const image = metadataRenderer.avatar?.thumbnails?.[0]?.url; + const description = metadataRenderer.description; + const playlistId = (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads; + + return { + channelName, + image, + description, + playlistId, + }; + }); + } + + // Get the appropriate playlist ID based on filterShorts setting + const playlistId = await (async () => { + if (userHandleData?.playlistId) { + const origPlaylistId = userHandleData.playlistId; + + return utils.getPlaylistWithShortsFilter(origPlaylistId, filterShorts); + } else { + const channelData = await utils.getChannelWithUsername(username, 'contentDetails', cache); + const items = channelData.data.items; + + if (!items) { + throw new NotFoundError(`The channel https://www.youtube.com/user/${username} does not exist.`); + } + + const channelId = items[0].id; + + return filterShorts ? utils.getPlaylistWithShortsFilter(channelId, filterShorts) : items[0].contentDetails.relatedPlaylists.uploads; + } + })(); + + const playlistItems = await utils.getPlaylistItems(playlistId, 'snippet', cache); + if (!playlistItems) { + throw new NotFoundError("This channel doesn't have any content."); + } + + return { + title: `${userHandleData?.channelName || username} - YouTube`, + link: username.startsWith('@') ? `https://www.youtube.com/${username}` : `https://www.youtube.com/user/${username}`, + description: userHandleData?.description || `YouTube user ${username}`, + image: userHandleData?.image, + item: playlistItems.data.items + .filter((d) => d.snippet.title !== 'Private video' && d.snippet.title !== 'Deleted video') + .map((item) => { + const snippet = item.snippet; + const videoId = snippet.resourceId.videoId; + const img = utils.getThumbnail(snippet.thumbnails); + return { + title: snippet.title, + description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)), + pubDate: parseDate(snippet.publishedAt), + link: `https://www.youtube.com/watch?v=${videoId}`, + author: snippet.videoOwnerChannelTitle, + image: img.url, + attachments: [ + { + url: getVideoUrl(videoId), + mime_type: 'text/html', + }, + ], + }; + }), + }; +}; diff --git a/lib/routes/youtube/api/youtubei.ts b/lib/routes/youtube/api/youtubei.ts new file mode 100644 index 000000000..e206eea1d --- /dev/null +++ b/lib/routes/youtube/api/youtubei.ts @@ -0,0 +1,48 @@ +import cache from '@/utils/cache'; +import { Innertube } from 'youtubei.js'; +import utils, { getVideoUrl } from '../utils'; +import { Data } from '@/types'; + +const innertubePromise = Innertube.create(); + +export const getChannelIdByUsername = (username: string) => + cache.tryGet(`youtube:getChannelIdByUsername:${username}`, async () => { + const innertube = await innertubePromise; + const navigationEndpoint = await innertube.resolveURL(`https://www.youtube.com/${username}`); + return navigationEndpoint.payload.browseId; + }); + +export const getDataByUsername = async ({ username, embed }: { username: string; embed: boolean; filterShorts: boolean }): Promise => { + const innertube = await innertubePromise; + const channelId = (await getChannelIdByUsername(username)) as string; + + const channel = await innertube.getChannel(channelId); + const videos = await channel.getVideos(); + + return { + title: `${channel.metadata.title || username} - YouTube`, + link: `https://www.youtube.com/${username}`, + image: channel.metadata.avatar?.[0].url, + description: channel.metadata.description, + + item: videos.videos + .filter((video) => 'video_id' in video) + .map((video) => { + const img = 'thumbnail' in video ? video.thumbnail?.[0].url : undefined; + + return { + title: video.title.text || `YouTube Video ${video.video_id}`, + description: 'description_snippet' in video ? utils.renderDescription(embed, video.video_id, img, utils.formatDescription(video.description_snippet?.toHTML())) : null, + link: `https://www.youtube.com/watch?v=${video.video_id}`, + author: typeof video.author === 'string' ? video.author : (video.author.name === 'N/A' ? undefined : video.author.name), + image: img, + attachments: [ + { + url: getVideoUrl(video.video_id), + mime_type: 'text/html', + }, + ], + }; + }), + }; +}; diff --git a/lib/routes/youtube/user.ts b/lib/routes/youtube/user.ts index a9efe79b7..117eaba27 100644 --- a/lib/routes/youtube/user.ts +++ b/lib/routes/youtube/user.ts @@ -1,12 +1,9 @@ import { Route, ViewType } from '@/types'; -import cache from '@/utils/cache'; -import utils, { getVideoUrl } from './utils'; import { config } from '@/config'; -import { parseDate } from '@/utils/parse-date'; -import ofetch from '@/utils/ofetch'; -import * as cheerio from 'cheerio'; import ConfigNotFoundError from '@/errors/types/config-not-found'; -import NotFoundError from '@/errors/types/not-found'; +import { getDataByUsername as getDataByUsernameYoutubei } from './api/youtubei'; +import { getDataByUsername as getDataByUsernameGoogle } from './api/google'; +import { callApi } from './utils'; export const route: Route = { path: '/user/:username/:routeParams?', @@ -64,84 +61,11 @@ async function handler(ctx) { const filterShortsStr = params.get('filterShorts'); const filterShorts = filterShortsStr === null || filterShortsStr === '' || filterShortsStr === 'true'; - let userHandleData; - if (username.startsWith('@')) { - userHandleData = await cache.tryGet(`youtube:handle:${username}`, async () => { - const link = `https://www.youtube.com/${username}`; - const response = await ofetch(link); - const $ = cheerio.load(response); - const ytInitialData = JSON.parse( - $('script') - .text() - .match(/ytInitialData = ({.*?});/)?.[1] || '{}' - ); - const metadataRenderer = ytInitialData.metadata.channelMetadataRenderer; + const data = await callApi({ + googleApi: getDataByUsernameGoogle, + youtubeiApi: getDataByUsernameYoutubei, + params: { username, embed, filterShorts }, + }); - const channelId = metadataRenderer.externalId; - const channelName = metadataRenderer.title; - const image = metadataRenderer.avatar?.thumbnails?.[0]?.url; - const description = metadataRenderer.description; - const playlistId = (await utils.getChannelWithId(channelId, 'contentDetails', cache)).data.items[0].contentDetails.relatedPlaylists.uploads; - - return { - channelName, - image, - description, - playlistId, - }; - }); - } - - // Get the appropriate playlist ID based on filterShorts setting - const playlistId = await (async () => { - if (userHandleData?.playlistId) { - const origPlaylistId = userHandleData.playlistId; - - return utils.getPlaylistWithShortsFilter(origPlaylistId, filterShorts); - } else { - const channelData = await utils.getChannelWithUsername(username, 'contentDetails', cache); - const items = channelData.data.items; - - if (!items) { - throw new NotFoundError(`The channel https://www.youtube.com/user/${username} does not exist.`); - } - - const channelId = items[0].id; - - return filterShorts ? utils.getPlaylistWithShortsFilter(channelId, filterShorts) : items[0].contentDetails.relatedPlaylists.uploads; - } - })(); - - const playlistItems = await utils.getPlaylistItems(playlistId, 'snippet', cache); - if (!playlistItems) { - throw new NotFoundError("This channel doesn't have any content."); - } - - return { - title: `${userHandleData?.channelName || username} - YouTube`, - link: username.startsWith('@') ? `https://www.youtube.com/${username}` : `https://www.youtube.com/user/${username}`, - description: userHandleData?.description || `YouTube user ${username}`, - image: userHandleData?.image, - item: playlistItems.data.items - .filter((d) => d.snippet.title !== 'Private video' && d.snippet.title !== 'Deleted video') - .map((item) => { - const snippet = item.snippet; - const videoId = snippet.resourceId.videoId; - const img = utils.getThumbnail(snippet.thumbnails); - return { - title: snippet.title, - description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)), - pubDate: parseDate(snippet.publishedAt), - link: `https://www.youtube.com/watch?v=${videoId}`, - author: snippet.videoOwnerChannelTitle, - image: img.url, - attachments: [ - { - url: getVideoUrl(videoId), - mime_type: 'text/html', - }, - ], - }; - }), - }; + return data; } diff --git a/lib/routes/youtube/utils.ts b/lib/routes/youtube/utils.ts index 61872eeab..ac1c37713 100644 --- a/lib/routes/youtube/utils.ts +++ b/lib/routes/youtube/utils.ts @@ -1,46 +1,8 @@ import { google } from 'googleapis'; -const { OAuth2 } = google.auth; import { art } from '@/utils/render'; import path from 'node:path'; import { config } from '@/config'; - -let count = 0; -const youtube = {}; -if (config.youtube && config.youtube.key) { - const keys = config.youtube.key.split(','); - - for (const [index, key] of keys.entries()) { - if (key) { - youtube[index] = google.youtube({ - version: 'v3', - auth: key, - }); - count = index + 1; - } - } -} - -let index = -1; -const exec = async (func) => { - let result; - for (let i = 0; i < count; i++) { - index++; - try { - // eslint-disable-next-line no-await-in-loop - result = await func(youtube[index % count]); - break; - } catch { - // console.error(error); - } - } - return result; -}; - -let youtubeOAuth2Client; -if (config.youtube && config.youtube.clientId && config.youtube.clientSecret && config.youtube.refreshToken) { - youtubeOAuth2Client = new OAuth2(config.youtube.clientId, config.youtube.clientSecret, 'https://developers.google.com/oauthplayground'); - youtubeOAuth2Client.setCredentials({ refresh_token: config.youtube.refreshToken }); -} +import { youtubeOAuth2Client, exec } from './api/google'; export const getPlaylistItems = (id, part, cache) => cache.tryGet( @@ -167,6 +129,17 @@ export const getPlaylistWithShortsFilter = (id: string, filterShorts = true): st return id; }; +export const callApi = async ({ googleApi, youtubeiApi, params }: { googleApi: (params: any) => Promise; youtubeiApi: (params: any) => Promise; params: any }): Promise => { + if (config.youtube?.key) { + try { + return await googleApi(params); + } catch { + return await youtubeiApi(params); + } + } + return await youtubeiApi(params); +}; + const youtubeUtils = { getPlaylistItems, getPlaylist, diff --git a/package.json b/package.json index 8bac96dc3..dfdf68b3f 100644 --- a/package.json +++ b/package.json @@ -136,6 +136,7 @@ "uuid": "11.1.0", "winston": "3.17.0", "xxhash-wasm": "1.1.0", + "youtubei.js": "^13.4.0", "zod": "3.24.4" }, "devDependencies": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 57bd5d11e..da0858a5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -280,6 +280,9 @@ importers: xxhash-wasm: specifier: 1.1.0 version: 1.1.0 + youtubei.js: + specifier: ^13.4.0 + version: 13.4.0 zod: specifier: 3.24.4 version: 3.24.4 @@ -1019,6 +1022,9 @@ packages: '@bcoe/v8-coverage@0.2.3': resolution: {integrity: sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==} + '@bufbuild/protobuf@2.4.0': + resolution: {integrity: sha512-RN9M76x7N11QRihKovEglEjjVCQEA9PRBVnDgk9xw8JHLrcUrp4FpAVSPSH91cNbcTft3u2vpLN4GMbiKY9PJw==} + '@bundled-es-modules/cookie@2.0.1': resolution: {integrity: sha512-8o+5fRPLNbjbdGRRmJj3h6Hh1AQJf2dk3qQ/5ZFb+PXkRNiSoMGGUKlsgLfrxneb72axVJyIYji64E2+nNfYyw==} @@ -1409,6 +1415,10 @@ packages: resolution: {integrity: sha512-ZAoA40rNMPwSm+AeHpCq8STiNAwzWLJuP8Xv4CHIc9wv/PSuExjMrmjfYNj682vW0OOiZ1HKxzvjQr9XZIisQA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/busboy@2.1.1': + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + '@fastify/otel@https://codeload.github.com/getsentry/fastify-otel/tar.gz/ae3088d65e286bdc94ac5d722573537d6a6671bb': resolution: {tarball: https://codeload.github.com/getsentry/fastify-otel/tar.gz/ae3088d65e286bdc94ac5d722573537d6a6671bb} version: 0.8.0 @@ -4362,6 +4372,9 @@ packages: jackspeak@3.4.3: resolution: {integrity: sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw==} + jintr@3.3.1: + resolution: {integrity: sha512-nnOzyhf0SLpbWuZ270Omwbj5LcXUkTcZkVnK8/veJXtSZOiATM5gMZMdmzN75FmTyj+NVgrGaPdH12zIJ24oIA==} + jiti@2.4.2: resolution: {integrity: sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A==} hasBin: true @@ -6210,6 +6223,10 @@ packages: undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@5.29.0: + resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} + engines: {node: '>=14.0'} + undici@6.21.3: resolution: {integrity: sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==} engines: {node: '>=18.17'} @@ -6590,6 +6607,9 @@ packages: resolution: {integrity: sha512-cYVsTjKl8b+FrnidjibDWskAv7UKOfcwaVZdp/it9n1s9fU3IkgDbhdIRKCW4JDsAlECJY0ytoVPT3sK6kideA==} engines: {node: '>=18'} + youtubei.js@13.4.0: + resolution: {integrity: sha512-+fmIZU/dWAjsROONrASy1REwVpy6umAPVuoNLr/4iNmZXl84LyBef0n3hrd1Vn9035EuINToGyQcBmifwUEemA==} + zod-to-json-schema@3.24.5: resolution: {integrity: sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==} peerDependencies: @@ -7342,6 +7362,8 @@ snapshots: '@bcoe/v8-coverage@0.2.3': {} + '@bufbuild/protobuf@2.4.0': {} + '@bundled-es-modules/cookie@2.0.1': dependencies: cookie: 0.7.2 @@ -7610,6 +7632,8 @@ snapshots: '@eslint/core': 0.13.0 levn: 0.4.1 + '@fastify/busboy@2.1.1': {} + '@fastify/otel@https://codeload.github.com/getsentry/fastify-otel/tar.gz/ae3088d65e286bdc94ac5d722573537d6a6671bb(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -10904,6 +10928,10 @@ snapshots: optionalDependencies: '@pkgjs/parseargs': 0.11.0 + jintr@3.3.1: + dependencies: + acorn: 8.14.1 + jiti@2.4.2: {} js-beautify@1.15.4: @@ -12911,6 +12939,10 @@ snapshots: undici-types@6.21.0: {} + undici@5.29.0: + dependencies: + '@fastify/busboy': 2.1.1 + undici@6.21.3: {} unicode-canonical-property-names-ecmascript@2.0.1: {} @@ -13275,6 +13307,13 @@ snapshots: yoctocolors-cjs@2.1.2: {} + youtubei.js@13.4.0: + dependencies: + '@bufbuild/protobuf': 2.4.0 + jintr: 3.3.1 + tslib: 2.8.1 + undici: 5.29.0 + zod-to-json-schema@3.24.5(zod@3.24.4): dependencies: zod: 3.24.4