fix(route): optimize transformer-circuits route (#18731)
* fix(route): optimize transformer-circuits route * fix(template): update link text from "阅读原文" to "Read Original" * fix(route): fix hard code issuses * fix(route): remove DOCTYPE declaration and HTML elements * fix(route):remove logger * feat(route): 为ES模块添加__dirname和__filename支持 * Update lib/routes/transformer-circuits/index.ts ---------
This commit is contained in:
ADAM HUANG
GitHub
parent
65a7d50285
commit
48169749e6
|
|
@ -1,7 +1,16 @@
|
|||
import { Route, DataItem } from '@/types';
|
||||
import cache from '@/utils/cache';
|
||||
import ofetch from '@/utils/ofetch';
|
||||
import { load } from 'cheerio';
|
||||
import { parseDate } from '@/utils/parse-date';
|
||||
import { art } from '@/utils/render';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import logger from '@/utils/logger';
|
||||
|
||||
// 为ES模块创建__dirname等价物
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
|
||||
// Define the main route path
|
||||
export const route: Route = {
|
||||
|
|
@ -27,8 +36,8 @@ async function handler() {
|
|||
const response = await ofetch(rootUrl);
|
||||
const $ = load(response);
|
||||
|
||||
// Get all the articles using .map() instead of .push()
|
||||
const articles: DataItem[] = $('.toc a')
|
||||
// Get all article links and basic info
|
||||
const articlePromises = $('.toc a')
|
||||
.toArray()
|
||||
.map((item) => {
|
||||
const $item = $(item);
|
||||
|
|
@ -40,40 +49,76 @@ async function handler() {
|
|||
if (currentElement.hasClass('paper') || currentElement.hasClass('note')) {
|
||||
const articleType = currentElement.hasClass('paper') ? 'Paper' : 'Note';
|
||||
|
||||
// Extract title
|
||||
// Extract title and metadata from the list page
|
||||
const title = currentElement.find('h3').text().trim();
|
||||
|
||||
// Extract author if available (papers have bylines, notes don't always have them)
|
||||
let author = '';
|
||||
const byline = currentElement.find('.byline');
|
||||
if (byline.length) {
|
||||
author = byline.text().trim();
|
||||
}
|
||||
// Extract description
|
||||
const description = currentElement.find('.description').text().trim();
|
||||
|
||||
// Get the article URL
|
||||
const href = currentElement.attr('href');
|
||||
const articleUrl = href ? (href.startsWith('http') ? href : `${rootUrl}/${href}`) : rootUrl;
|
||||
|
||||
// Create article object
|
||||
return {
|
||||
title,
|
||||
link: articleUrl,
|
||||
pubDate: parseDate(currentDate, 'MMMM YYYY'),
|
||||
author,
|
||||
description: `${articleType}: ${description}`,
|
||||
category: ['AI', 'Machine Learning', 'Anthropic', 'Transformer Circuits'],
|
||||
};
|
||||
// Cache the whole article object instead of just the content
|
||||
return cache.tryGet(articleUrl, async () => {
|
||||
const fullContent = await fetchArticleContent(articleUrl);
|
||||
return {
|
||||
title,
|
||||
link: articleUrl,
|
||||
pubDate: parseDate(currentDate, 'MMMM YYYY'),
|
||||
author,
|
||||
description: fullContent || `${articleType}: ${description}`,
|
||||
category: ['AI', 'Machine Learning', 'Anthropic', 'Transformer Circuits'],
|
||||
};
|
||||
});
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter(Boolean) as DataItem[];
|
||||
});
|
||||
|
||||
// Wait for all article fetches to complete
|
||||
const articlesWithContent = (await Promise.all(articlePromises)).filter(Boolean) as DataItem[];
|
||||
|
||||
return {
|
||||
title: 'Transformer Circuits Thread',
|
||||
link: rootUrl,
|
||||
item: articles,
|
||||
item: articlesWithContent,
|
||||
description: 'Research on reverse engineering transformer language models into human-understandable programs',
|
||||
};
|
||||
}
|
||||
|
||||
// Function to fetch and parse article content
|
||||
async function fetchArticleContent(url) {
|
||||
try {
|
||||
const response = await ofetch(url);
|
||||
const $ = load(response);
|
||||
|
||||
// Remove navigation and other unnecessary elements
|
||||
$('.article-header, .tooltip, modal, script, style, d-front-matter, .visual-toc').remove();
|
||||
|
||||
// Get the main content - d-article is the custom tag used by this site
|
||||
let content = $('d-article').html();
|
||||
|
||||
// If d-article not found, try more specific fallback selectors
|
||||
if (!content) {
|
||||
// Try to find content in common article containers, but avoid selecting the entire body
|
||||
content = $('main article, .article-content, .post-content, .content-area').html() || $('.content, .article, .post').html() || $('main').html();
|
||||
|
||||
// If still no content found, log a warning
|
||||
if (!content) {
|
||||
logger.warn(`No suitable content container found for ${url}`);
|
||||
content = `<p>Could not extract content. Please visit <a href="${url}">the original page</a>.</p>`;
|
||||
}
|
||||
}
|
||||
|
||||
// Create an HTML fragment (not a full document) for the RSS description
|
||||
return art(path.join(__dirname, 'templates/article.art'), {
|
||||
content,
|
||||
link: url,
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
logger.error(`Error fetching article content from ${url}: ${errorMessage}`);
|
||||
return null; // Return null on error, we'll fall back to description
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,62 @@
|
|||
<style>
|
||||
.content-wrapper {
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
||||
line-height: 1.6;
|
||||
color: #333;
|
||||
}
|
||||
img {
|
||||
max-width: 100%;
|
||||
height: auto;
|
||||
}
|
||||
pre, code {
|
||||
background-color: #f5f5f5;
|
||||
border-radius: 3px;
|
||||
padding: 0.2em 0.4em;
|
||||
overflow-x: auto;
|
||||
}
|
||||
a {
|
||||
color: #0366d6;
|
||||
text-decoration: none;
|
||||
}
|
||||
a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
h1, h2, h3, h4, h5, h6 {
|
||||
margin-top: 24px;
|
||||
margin-bottom: 16px;
|
||||
font-weight: 600;
|
||||
line-height: 1.25;
|
||||
}
|
||||
p, ul, ol {
|
||||
margin-bottom: 16px;
|
||||
}
|
||||
.read-original {
|
||||
margin-top: 30px;
|
||||
margin-bottom: 30px;
|
||||
text-align: center;
|
||||
padding: 10px;
|
||||
background-color: #f7f7f7;
|
||||
border-radius: 4px;
|
||||
}
|
||||
/* Support for custom elements used on transformer-circuits website */
|
||||
d-figure, figure {
|
||||
margin: 20px 0;
|
||||
text-align: center;
|
||||
}
|
||||
d-byline {
|
||||
font-size: 0.9em;
|
||||
color: #666;
|
||||
margin: 15px 0;
|
||||
}
|
||||
.gdoc-image img {
|
||||
max-width: 100%;
|
||||
display: block;
|
||||
margin: 0 auto;
|
||||
}
|
||||
</style>
|
||||
<div class="content-wrapper">
|
||||
{{@ content }}
|
||||
</div>
|
||||
<div class="read-original">
|
||||
<a href="{{ link }}" target="_blank">Read Original</a>
|
||||
</div>
|
||||
Loading…
Reference in New Issue