test: add full routes test (#14934)

* test: add critical routes test

* fix: youtube channels config

* test: add full routes test

* test: output full routes test result

* test: build routes before test

* test: use one reporter

* test: fix cache test errors

* test: fix cache test errors

* chore: puppeteerrc.cjs

* chore: puppeteerrc.cjs

* chore: puppeteerrc.cjs

* fix: build script

* chore: puppeteerrc rename

* test: route test names

* chore: pass test result to docs
This commit is contained in:
DIYgod 2024-03-24 11:00:03 +00:00 committed by GitHub
parent 2fc3b3a336
commit b774c60858
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 247 additions and 198 deletions

View File

@ -33,7 +33,11 @@ jobs:
- name: Install dependencies (yarn)
run: pnpm i
- name: Build assets
run: npm run build
run: pnpm build
- name: Build full routes test result
run: pnpm vitest:fullroutes
- name: Build docs
run: pnpm build:docs
- name: Commit files
run: |
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"

View File

@ -5,7 +5,6 @@ on:
branches-ignore:
- 'dependabot/**'
paths:
- 'test/**'
- 'lib/**'
- 'package.json'
- 'pnpm-lock.yaml'
@ -43,6 +42,8 @@ jobs:
run: pnpm i
- name: Run postinstall script for dependencies
run: pnpm rb
- name: Build routes
run: pnpm build
- name: Test all and generate coverage
run: pnpm run vitest:coverage
env:
@ -84,6 +85,8 @@ jobs:
run: pnpm i
- name: Run postinstall script for dependencies
run: pnpm rb
- name: Build routes
run: pnpm build
- name: Install Chromium
if: ${{ matrix.chromium.dependency != '' }}
# 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means

View File

@ -84,7 +84,7 @@ FROM node:21-bookworm-slim AS chromium-downloader
# Yeah, downloading Chromium never needs those dependencies below.
WORKDIR /app
COPY ./.puppeteerrc.js /app/
COPY ./.puppeteerrc.cjs /app/
COPY --from=dep-version-parser /ver/.puppeteer_version /app/.puppeteer_version
ARG TARGETPLATFORM

View File

@ -1,23 +1,16 @@
import { describe, expect, it, vi, afterEach, afterAll, beforeAll } from 'vitest';
import { describe, expect, it, vi, afterEach } from 'vitest';
import Parser from 'rss-parser';
import wait from '@/utils/wait';
process.env.CACHE_EXPIRE = '1';
process.env.CACHE_CONTENT_EXPIRE = '3';
const parser = new Parser();
beforeAll(() => {
process.env.CACHE_EXPIRE = '1';
process.env.CACHE_CONTENT_EXPIRE = '3';
});
afterEach(() => {
delete process.env.CACHE_TYPE;
vi.resetModules();
});
afterAll(() => {
delete process.env.CACHE_EXPIRE;
});
describe('cache', () => {
it('memory', async () => {
process.env.CACHE_TYPE = 'memory';

View File

@ -1,55 +1,8 @@
import { describe, expect, it, afterAll } from 'vitest';
process.env.SOCKET = 'socket';
import { describe, expect, it } from 'vitest';
import app from '@/app';
import Parser from 'rss-parser';
const parser = new Parser();
import { config } from '@/config';
afterAll(() => {
delete process.env.SOCKET;
});
async function checkRSS(response) {
const checkDate = (date) => {
expect(date).toEqual(expect.any(String));
expect(Date.parse(date)).toEqual(expect.any(Number));
expect(Date.now() - +new Date(date)).toBeGreaterThan(-1000 * 60 * 60 * 24 * 5);
expect(Date.now() - +new Date(date)).toBeLessThan(1000 * 60 * 60 * 24 * 30 * 12 * 10);
};
const parsed = await parser.parseString(await response.text());
expect(parsed).toEqual(expect.any(Object));
expect(parsed.title).toEqual(expect.any(String));
expect(parsed.title).not.toBe('RSSHub');
expect(parsed.description).toEqual(expect.any(String));
expect(parsed.link).toEqual(expect.any(String));
expect(parsed.lastBuildDate).toEqual(expect.any(String));
expect(parsed.ttl).toEqual(Math.trunc(config.cache.routeExpire / 60) + '');
expect(parsed.items).toEqual(expect.any(Array));
checkDate(parsed.lastBuildDate);
// check items
const guids: (string | undefined)[] = [];
for (const item of parsed.items) {
expect(item).toEqual(expect.any(Object));
expect(item.title).toEqual(expect.any(String));
expect(item.link).toEqual(expect.any(String));
expect(item.content).toEqual(expect.any(String));
expect(item.guid).toEqual(expect.any(String));
if (item.pubDate) {
expect(item.pubDate).toEqual(expect.any(String));
checkDate(item.pubDate);
}
// guid must be unique
expect(guids).not.toContain(item.guid);
guids.push(item.guid);
}
}
describe('router', () => {
describe('registry', () => {
// root
it(`/`, async () => {
const response = await app.request('/');
@ -58,14 +11,6 @@ describe('router', () => {
expect(response.headers.get('cache-control')).toBe('no-cache');
});
// route
it(`/test/1`, async () => {
const response = await app.request('/test/1');
expect(response.status).toBe(200);
await checkRSS(response);
});
// robots.txt
it('/robots.txt', async () => {
config.disallowRobot = false;

View File

@ -7,8 +7,6 @@ import { serveStatic } from '@hono/node-server/serve-static';
import index from '@/routes/index';
import robotstxt from '@/routes/robots.txt';
import { namespace as testNamespace } from './routes/test/namespace';
import { route as testRoute } from '@/routes/test/index';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@ -27,18 +25,9 @@ let namespaces: Record<
switch (process.env.NODE_ENV) {
case 'test':
modules = {
'/test/namespace.ts': {
namespace: testNamespace,
},
'/test/index.ts': {
route: testRoute,
},
};
break;
case 'production':
// eslint-disable-next-line n/no-unpublished-require
namespaces = require('../assets/build/routes.json');
// @ts-expect-error
namespaces = await import('../assets/build/routes.json');
break;
default:
modules = directoryImport({

78
lib/routes.test.ts Normal file
View File

@ -0,0 +1,78 @@
import { describe, expect, it } from 'vitest';
import app from '@/app';
import Parser from 'rss-parser';
const parser = new Parser();
import { config } from '@/config';
process.env.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN = 'true';
const routes = {
'/test/:id': '/test/1',
};
if (process.env.FULL_ROUTES_TEST) {
const { namespaces } = await import('@/registry');
for (const namespace in namespaces) {
for (const route in namespaces[namespace].routes) {
const requireConfig = namespaces[namespace].routes[route].features?.requireConfig;
let configs;
if (typeof requireConfig !== 'boolean') {
configs = requireConfig
?.filter((config) => !config.optional)
.map((config) => config.name)
.filter((name) => name !== 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN');
}
if (namespaces[namespace].routes[route].example && !configs?.length) {
routes[`/${namespace}${route}`] = namespaces[namespace].routes[route].example;
}
}
}
}
async function checkRSS(response) {
const checkDate = (date) => {
expect(date).toEqual(expect.any(String));
expect(Date.parse(date)).toEqual(expect.any(Number));
expect(Date.now() - +new Date(date)).toBeGreaterThan(-1000 * 60 * 60 * 24 * 5);
expect(Date.now() - +new Date(date)).toBeLessThan(1000 * 60 * 60 * 24 * 30 * 12 * 10);
};
const parsed = await parser.parseString(await response.text());
expect(parsed).toEqual(expect.any(Object));
expect(parsed.title).toEqual(expect.any(String));
expect(parsed.title).not.toBe('RSSHub');
expect(parsed.description).toEqual(expect.any(String));
expect(parsed.link).toEqual(expect.any(String));
expect(parsed.lastBuildDate).toEqual(expect.any(String));
expect(parsed.ttl).toEqual(Math.trunc(config.cache.routeExpire / 60) + '');
expect(parsed.items).toEqual(expect.any(Array));
checkDate(parsed.lastBuildDate);
// check items
const guids: (string | undefined)[] = [];
for (const item of parsed.items) {
expect(item).toEqual(expect.any(Object));
expect(item.title).toEqual(expect.any(String));
expect(item.link).toEqual(expect.any(String));
expect(item.content).toEqual(expect.any(String));
expect(item.guid).toEqual(expect.any(String));
if (item.pubDate) {
expect(item.pubDate).toEqual(expect.any(String));
checkDate(item.pubDate);
}
// guid must be unique
expect(guids).not.toContain(item.guid);
guids.push(item.guid);
}
}
describe('routes', () => {
for (const route in routes) {
it.concurrent(route, async () => {
const response = await app.request(routes[route]);
expect(response.status).toBe(200);
await checkRSS(response);
});
}
});

View File

@ -1,4 +1,5 @@
import got from '@/utils/got';
import type { Route } from '@/types';
export const route: Route = {
path: '/dynamic/:uid?',

View File

@ -121,7 +121,6 @@ async function handler(ctx) {
headers: {
Referer: `https://space.bilibili.com/${uid}/`,
},
transformResponse: [(data) => data],
});
const cards = JSONbig.parse(response.body).data.cards;

View File

@ -79,13 +79,9 @@ async function handler(ctx) {
break;
case 'cache': {
const description = await cache.tryGet(
'test',
() => ({
text: `Cache${++cacheIndex}`,
}),
config.cache.routeExpire * 2
);
const description = await cache.tryGet('test', () => ({
text: `Cache${++cacheIndex}`,
}));
item.push({
title: 'Cache Title',
description: description.text,

View File

@ -10,7 +10,12 @@ export const route: Route = {
example: '/youtube/channel/UCDwDMPOZfxVV0x_dz0eQ8KQ',
parameters: { id: 'YouTube channel id', embed: 'Default to embed the video, set to any value to disable embedding' },
features: {
requireConfig: false,
requireConfig: [
{
name: 'YOUTUBE_KEY',
description: ' YouTube API Key, support multiple keys, split them with `,`, [API Key application](https://console.developers.google.com/)',
},
],
requirePuppeteer: false,
antiCrawler: false,
supportBT: false,

View File

@ -19,20 +19,23 @@
"files": [
"lib"
],
"type": "module",
"scripts": {
"build": "tsx scripts/workflow/build-routes.ts",
"dev": "cross-env NODE_ENV=dev tsx watch --no-cache lib/index.ts",
"dev:cache": "cross-env NODE_ENV=production tsx watch lib/index.ts",
"build:docs": "tsx scripts/workflow/build-docs.ts",
"dev": "NODE_ENV=dev tsx watch --no-cache lib/index.ts",
"dev:cache": "NODE_ENV=production tsx watch lib/index.ts",
"format": "eslint --cache --fix \"**/*.{ts,js,yml}\" && prettier \"**/*.{ts,js,json}\" --write",
"format:check": "eslint --cache \"**/*.{ts,js,yml}\" && prettier \"**/*.{ts,js,json}\" --check",
"format:staged": "lint-staged",
"vitest": "cross-env NODE_ENV=test vitest",
"vitest:coverage": "cross-env NODE_ENV=test vitest --coverage.enabled --reporter=junit",
"vitest:watch": "cross-env NODE_ENV=test vitest --watch",
"vitest": "NODE_ENV=test vitest",
"vitest:fullroutes": "NODE_ENV=test FULL_ROUTES_TEST=true vitest --reporter=json --reporter=default --outputFile=\"./assets/build/test-full-routes.json\" routes",
"vitest:coverage": "NODE_ENV=test vitest --coverage.enabled --reporter=junit",
"vitest:watch": "NODE_ENV=test vitest --watch",
"lint": "eslint --cache .",
"prepare": "husky || true",
"profiling": "NODE_ENV=production tsx --prof lib/index.ts",
"start": "cross-env NODE_ENV=production tsx lib/index.ts",
"start": "NODE_ENV=production tsx lib/index.ts",
"test": "npm run format:check && npm run vitest:coverage"
},
"lint-staged": {
@ -147,7 +150,6 @@
"@typescript-eslint/parser": "7.3.1",
"@vercel/nft": "0.26.4",
"@vitest/coverage-v8": "1.4.0",
"cross-env": "7.0.3",
"eslint": "8.57.0",
"eslint-config-prettier": "9.1.0",
"eslint-nibble": "8.1.0",

View File

@ -301,9 +301,6 @@ devDependencies:
'@vitest/coverage-v8':
specifier: 1.4.0
version: 1.4.0(vitest@1.4.0)
cross-env:
specifier: 7.0.3
version: 7.0.3
eslint:
specifier: 8.57.0
version: 8.57.0
@ -3901,14 +3898,6 @@ packages:
typescript: 5.4.3
dev: false
/cross-env@7.0.3:
resolution: {integrity: sha512-+/HKd6EgcQCJGh2PSjZuUitQBQynKor4wrFbRg4DtAgS1aWO+gU52xpH7M9ScGgXSYmAVS9bIJ8EzuaGw0oNAw==}
engines: {node: '>=10.14', npm: '>=6', yarn: '>=1'}
hasBin: true
dependencies:
cross-spawn: 7.0.3
dev: true
/cross-spawn@5.1.0:
resolution: {integrity: sha512-pTgQJ5KC0d2hcY8eyL1IzlBPYjTkyH72XRZPnLyKus2mBfNjQs3klqbJU2VILqZryAZUt9JOb3h/mWMy23/f5A==}
dependencies:

View File

@ -0,0 +1,122 @@
import { namespaces } from '../../lib/registry';
import fs from 'node:fs';
import * as path from 'node:path';
import { categories } from './data';
import fullTests from '../../assets/build/test-full-routes.json';
import { getCurrentPath } from '../../lib/utils/helpers';
const testResult = fullTests.testResults[0].assertionResults;
const __dirname = getCurrentPath(import.meta.url);
const docs = {};
for (const namespace in namespaces) {
let defaultCategory = namespaces[namespace].categories?.[0];
if (!defaultCategory) {
for (const path in namespaces[namespace].routes) {
if (namespaces[namespace].routes[path].categories) {
defaultCategory = namespaces[namespace].routes[path].categories[0];
break;
}
}
}
if (!defaultCategory) {
defaultCategory = 'other';
}
for (const path in namespaces[namespace].routes) {
const realPath = `/${namespace}${path}`;
const data = namespaces[namespace].routes[path];
const categories = data.categories || namespaces[namespace].categories || [defaultCategory];
// docs.json
for (const category of categories) {
if (!docs[category]) {
docs[category] = {};
}
if (!docs[category][namespace]) {
docs[category][namespace] = {
routes: {},
};
}
docs[category][namespace].name = namespaces[namespace].name;
docs[category][namespace].url = namespaces[namespace].url;
docs[category][namespace].description = namespaces[namespace].description;
docs[category][namespace].routes[realPath] = data;
}
}
}
// Generate markdown
const pinyinCompare = new Intl.Collator('zh-Hans-CN-u-co-pinyin').compare;
const isASCII = (str) => /^[\u0000-\u007F]*$/.test(str);
function generateMd(lang) {
const md = {};
for (const category in docs) {
const nameObj = categories.find((c) => c.link.includes(category));
if (!nameObj) {
throw new Error(`Category not found: ${category}, please double check your spelling.`);
}
md[category] = `# ${`${nameObj.icon} ${nameObj[lang]}`}\n\n`;
const namespaces = Object.keys(docs[category]).sort((a, b) => {
const aname = docs[category][a].name[0];
const bname = docs[category][b].name[0];
const ia = isASCII(aname);
const ib = isASCII(bname);
if (ia && ib) {
return aname.toLowerCase() < bname.toLowerCase() ? -1 : 1;
} else if (ia || ib) {
return ia > ib ? -1 : 1;
} else {
return pinyinCompare(aname, bname);
}
});
for (const namespace of namespaces) {
if (docs[category][namespace].name === 'Unknown') {
docs[category][namespace].name = namespace;
}
md[category] += `## ${docs[category][namespace].name || namespace} ${docs[category][namespace].url ? `<Site url="${docs[category][namespace].url}"/>` : ''}\n\n`;
if (docs[category][namespace].description) {
md[category] += `${docs[category][namespace].description}\n\n`;
}
const realPaths = Object.keys(docs[category][namespace].routes).sort((a, b) => {
const aname = docs[category][namespace].routes[a].name[0];
const bname = docs[category][namespace].routes[b].name[0];
const ia = isASCII(aname);
const ib = isASCII(bname);
if (ia && ib) {
return aname.toLowerCase() < bname.toLowerCase() ? -1 : 1;
} else if (ia || ib) {
return ia > ib ? -1 : 1;
} else {
return pinyinCompare(aname, bname);
}
});
for (const realPath of realPaths) {
const data = docs[category][namespace].routes[realPath];
const test = testResult.find((t) => t.title === realPath);
const parsedTest = test
? {
code: test.status === 'passed' ? 0 : 1,
message: test.failureMessages?.[0],
}
: undefined;
md[category] += `### ${data.name} ${data.url || docs[category][namespace].url ? `<Site url="${data.url || docs[category][namespace].url}" size="sm" />` : ''}\n\n`;
md[category] += `<Route namespace="${namespace}" :data='${JSON.stringify(data).replaceAll(`'`, '&#39;')}' :test='${JSON.stringify(parsedTest)}' />\n\n`;
if (data.description) {
md[category] += `${data.description}\n\n`;
}
}
}
}
fs.mkdirSync(path.join(__dirname, `../../assets/build/docs/${lang}`), { recursive: true });
for (const category in md) {
fs.writeFileSync(path.join(__dirname, `../../assets/build/docs/${lang}/${category}.md`), md[category]);
}
}
generateMd('en');
generateMd('zh');

View File

@ -4,7 +4,9 @@ import { parse } from 'tldts';
import fs from 'node:fs';
import * as path from 'node:path';
import toSource from 'tosource';
import { categories } from './data';
import { getCurrentPath } from '../../lib/utils/helpers';
const __dirname = getCurrentPath(import.meta.url);
const maintainers: Record<string, string[]> = {};
const radar: {
@ -13,7 +15,6 @@ const radar: {
[subdomain: string]: RadarItem[] | string;
};
} = {};
const docs = {};
for (const namespace in namespaces) {
let defaultCategory = namespaces[namespace].categories?.[0];
@ -63,21 +64,6 @@ for (const namespace in namespaces) {
}
}
}
// docs.json
for (const category of categories) {
if (!docs[category]) {
docs[category] = {};
}
if (!docs[category][namespace]) {
docs[category][namespace] = {
routes: {},
};
}
docs[category][namespace].name = namespaces[namespace].name;
docs[category][namespace].url = namespaces[namespace].url;
docs[category][namespace].description = namespaces[namespace].description;
docs[category][namespace].routes[realPath] = data;
}
}
}
@ -85,71 +71,3 @@ fs.writeFileSync(path.join(__dirname, '../../assets/build/radar-rules.json'), JS
fs.writeFileSync(path.join(__dirname, '../../assets/build/radar-rules.js'), `(${toSource(radar)})`);
fs.writeFileSync(path.join(__dirname, '../../assets/build/maintainers.json'), JSON.stringify(maintainers, null, 2));
fs.writeFileSync(path.join(__dirname, '../../assets/build/routes.json'), JSON.stringify(namespaces, null, 2));
// Generate markdown
const pinyinCompare = new Intl.Collator('zh-Hans-CN-u-co-pinyin').compare;
const isASCII = (str) => /^[\u0000-\u007F]*$/.test(str);
function generateMd(lang) {
const md = {};
for (const category in docs) {
const nameObj = categories.find((c) => c.link.includes(category));
if (!nameObj) {
throw new Error(`Category not found: ${category}, please double check your spelling.`);
}
md[category] = `# ${`${nameObj.icon} ${nameObj[lang]}`}\n\n`;
const namespaces = Object.keys(docs[category]).sort((a, b) => {
const aname = docs[category][a].name[0];
const bname = docs[category][b].name[0];
const ia = isASCII(aname);
const ib = isASCII(bname);
if (ia && ib) {
return aname.toLowerCase() < bname.toLowerCase() ? -1 : 1;
} else if (ia || ib) {
return ia > ib ? -1 : 1;
} else {
return pinyinCompare(aname, bname);
}
});
for (const namespace of namespaces) {
if (docs[category][namespace].name === 'Unknown') {
docs[category][namespace].name = namespace;
}
md[category] += `## ${docs[category][namespace].name || namespace} ${docs[category][namespace].url ? `<Site url="${docs[category][namespace].url}"/>` : ''}\n\n`;
if (docs[category][namespace].description) {
md[category] += `${docs[category][namespace].description}\n\n`;
}
const realPaths = Object.keys(docs[category][namespace].routes).sort((a, b) => {
const aname = docs[category][namespace].routes[a].name[0];
const bname = docs[category][namespace].routes[b].name[0];
const ia = isASCII(aname);
const ib = isASCII(bname);
if (ia && ib) {
return aname.toLowerCase() < bname.toLowerCase() ? -1 : 1;
} else if (ia || ib) {
return ia > ib ? -1 : 1;
} else {
return pinyinCompare(aname, bname);
}
});
for (const realPath of realPaths) {
const data = docs[category][namespace].routes[realPath];
md[category] += `### ${data.name} ${data.url || docs[category][namespace].url ? `<Site url="${data.url || docs[category][namespace].url}" size="sm" />` : ''}\n\n`;
md[category] += `<Route namespace="${namespace}" :data='${JSON.stringify(data).replaceAll(`'`, '&#39;')}' />\n\n`;
if (data.description) {
md[category] += `${data.description}\n\n`;
}
}
}
}
fs.mkdirSync(path.join(__dirname, `../../assets/build/docs/${lang}`), { recursive: true });
for (const category in md) {
fs.writeFileSync(path.join(__dirname, `../../assets/build/docs/${lang}/${category}.md`), md[category]);
}
}
generateMd('en');
generateMd('zh');

View File

@ -2,7 +2,7 @@
"compilerOptions": {
"target": "ESNext",
"module": "ESNext",
"moduleResolution": "Bundler",
"moduleResolution": "node",
"strict": true,
"jsx": "react-jsx",
"jsxImportSource": "hono/jsx",
@ -11,7 +11,12 @@
},
"esModuleInterop": true,
"noImplicitAny": false,
"outDir": "./dist"
"outDir": "./dist",
"skipLibCheck": true,
"noEmit": true,
"incremental": true,
"resolveJsonModule": true,
"isolatedModules": true
},
"include": ["./lib/**/*", "./api/vercel.ts"],
"exclude": ["node_modules", "*.test.*"]