/* eslint-disable global-require */ const fs = require('node:fs'); const path = require('node:path'); const { externalLinkProcessor } = require('./tools/utils/externalLink'); const GROUP_ORDER = [ 'Autoscaling', 'Browser management', 'Configuration', 'Crawlers', 'Crawling contexts', 'Errors', 'Event data', 'Event managers', 'Functions', 'HTTP clients', 'HTTP parsers', 'Request loaders', 'Session management', 'Statistics', 'Storage clients', 'Storage data', 'Storages', 'Other', ]; const groupSort = (g1, g2) => { if (GROUP_ORDER.includes(g1) && GROUP_ORDER.includes(g2)) { return GROUP_ORDER.indexOf(g1) - GROUP_ORDER.indexOf(g2); } return g1.localeCompare(g2); }; /** @type {Partial} */ module.exports = { title: 'Crawlee for Python · Fast, reliable Python web crawlers.', url: 'https://crawlee.dev', baseUrl: '/python/', trailingSlash: false, organizationName: 'apify', projectName: 'crawlee-python', scripts: [ '/python/js/custom.js', '/crawlee-python/js/custom.js', ], githubHost: 'github.com', future: { faster: true, v4: { removeLegacyPostBuildHeadAttribute: true, useCssCascadeLayers: false, // this breaks styles on homepage and link colors everywhere }, }, headTags: [ // Intercom messenger { tagName: 'script', innerHTML: `window.intercomSettings={api_base:"https://api-iam.intercom.io",app_id:"kod1r788"};`, attributes: {}, }, // Intercom messenger { tagName: 'script', innerHTML: `(function(){var w=window;var ic=w.Intercom;if(typeof ic==="function"){ic('reattach_activator');ic('update',w.intercomSettings);}else{var d=document;var i=function(){i.c(arguments);};i.q=[];i.c=function(args){i.q.push(args);};w.Intercom=i;var l=function(){var s=d.createElement('script');s.type='text/javascript';s.async=true;s.src='https://widget.intercom.io/widget/kod1r788';var x=d.getElementsByTagName('script')[0];x.parentNode.insertBefore(s,x);};if(document.readyState==='complete'){l();}else if(w.attachEvent){w.attachEvent('onload',l);}else{w.addEventListener('load',l,false);}}})()`, attributes: {}, }, ], favicon: 'img/favicon.ico', customFields: { markdownOptions: { html: true, }, gaGtag: true, repoUrl: 'https://github.com/apify/crawlee-python', }, onBrokenLinks: 'throw', markdown: { mermaid: true, hooks: { onBrokenMarkdownLinks: 'throw', }, }, themes: [ '@docusaurus/theme-mermaid', ], presets: /** @type {import('@docusaurus/types').PresetConfig[]} */ ([ [ '@docusaurus/preset-classic', /** @type {import('@docusaurus/preset-classic').Options} */ ({ docs: { showLastUpdateAuthor: true, showLastUpdateTime: true, path: '../docs', sidebarPath: './sidebars.js', rehypePlugins: [externalLinkProcessor], editUrl: (doc) => { return `https://github.com/apify/crawlee-python/edit/master/website/${doc.versionDocsDirPath}/${doc.docPath}`; }, }, theme: { customCss: '/src/css/custom.css', }, }), ], ]), plugins: [ [ '@apify/docusaurus-plugin-typedoc-api', { projectRoot: '.', changelogs: false, readmes: false, packages: [{ path: '.' }], typedocOptions: { excludeExternals: false, }, sortSidebar: groupSort, routeBasePath: 'api', python: true, pythonOptions: { pythonModulePath: path.join(__dirname, '../src/crawlee'), moduleShortcutsPath: path.join(__dirname, 'module_shortcuts.json'), }, }, ], // [ // '@docusaurus/plugin-client-redirects', // { // redirects: [ // { // from: '/docs', // to: '/docs/quick-start', // }, // { // from: '/docs/next', // to: '/docs/next/quick-start', // }, // { // from: '/docs/guides/environment-variables', // to: '/docs/guides/configuration', // }, // { // from: '/docs/guides/getting-started', // to: '/docs/introduction', // }, // { // from: '/docs/guides/apify-platform', // to: '/docs/deployment/apify-platform', // }, // ], // createRedirects(existingPath) { // if (!existingPath.endsWith('/')) { // return `${existingPath}/`; // } // // return undefined; // Return a falsy value: no redirect created // }, // }, // ], [ 'docusaurus-gtm-plugin', { id: 'GTM-5P7MCS7', }, ], [ '@signalwire/docusaurus-plugin-llms-txt', { enableDescriptions: false, content: { includeVersionedDocs: false, enableLlmsFullTxt: true, relativePaths: false, excludeRoutes: [ '/python/api/[0-9]*/**', '/python/api/[0-9]*', '/python/api/next/**', '/python/api/next', ], }, }, ], async function runnableCodeBlock() { return { name: 'runnable-code-block', configureWebpack() { return { resolveLoader: { alias: { 'roa-loader': require.resolve(`${__dirname}/roa-loader/`), }, }, }; }, }; }, // skipping svgo for animated crawlee logo async function doNotUseSVGO() { return { name: 'docusaurus-svgo', configureWebpack(config) { // find the svg rule const svgRule = config.module.rules.find((r) => typeof r === 'object' && r.test.toString() === '/\\.svg$/i'); // find the svgr loader const svgrLoader = svgRule?.oneOf?.[0]; // make copy of svgr loader and disable svgo const svgrLoaderCopy = JSON.parse(JSON.stringify(svgrLoader)); // include only animated logo svgrLoaderCopy.include = /animated-crawlee-logo/; // turn off svgo svgrLoaderCopy.use[0].options.svgo = false; // insert the copy after the original svgr loader svgRule.oneOf.splice(1, 0, svgrLoaderCopy); // exclude animated logo from the first svgr loader (with svgo enabled) svgrLoader.exclude = /animated-crawlee-logo/; return { mergeStrategy: { 'module.rules': 'replace', }, module: { rules: config.module.rules, }, }; }, }; }, // Copy root CHANGELOG.md to docs/ and all versioned_docs/ so every // doc version displays the same (latest) changelog — not a snapshot. function changelogFromRoot() { const sourceChangelog = path.join(__dirname, '..', 'CHANGELOG.md'); return { name: 'changelog-from-root', async loadContent() { if (!fs.existsSync(sourceChangelog)) return; const changelog = fs.readFileSync(sourceChangelog, 'utf-8'); const docsDir = path.join(__dirname, '..', 'docs'); const versionedDocsDir = path.join(__dirname, 'versioned_docs'); const targetDirs = [docsDir]; if (fs.existsSync(versionedDocsDir)) { for (const version of fs.readdirSync(versionedDocsDir)) { targetDirs.push(path.join(versionedDocsDir, version)); } } for (const dir of targetDirs) { fs.writeFileSync(path.join(dir, 'changelog.md'), changelog); } }, getPathsToWatch() { return [sourceChangelog]; }, }; }, [ path.resolve(__dirname, 'src/plugins/docusaurus-plugin-segment'), { writeKey: process.env.SEGMENT_TOKEN, allowedInDev: false, }, ], ], themeConfig: /** @type {import('@docusaurus/preset-classic').ThemeConfig} */ ({ docs: { versionPersistence: 'localStorage', sidebar: { hideable: true, }, }, navbar: { hideOnScroll: true, logo: { src: 'img/crawlee-python-light.svg', srcDark: 'img/crawlee-python-dark.svg', }, title: 'Crawlee for Python', items: [ { type: 'doc', docId: 'quick-start/quick-start', label: 'Docs', position: 'left', }, { type: 'doc', docId: '/examples', label: 'Examples', position: 'left', }, { type: 'custom-api', label: 'API', position: 'left', activeBaseRegex: 'api/(?!.*/changelog)', }, { type: 'doc', label: 'Changelog', docId: 'changelog', className: 'changelog', }, { href: 'https://crawlee.dev/blog', target: '_self', rel: 'dofollow', label: 'Blog', position: 'left', }, { type: 'docsVersionDropdown', position: 'right', dropdownItemsBefore: [], dropdownItemsAfter: [], }, ], }, colorMode: { defaultMode: 'light', disableSwitch: false, respectPrefersColorScheme: true, }, prism: { defaultLanguage: 'typescript', theme: require('prism-react-renderer').themes.github, darkTheme: require('prism-react-renderer').themes.dracula, additionalLanguages: ['docker', 'log', 'bash', 'diff', 'json'], }, metadata: [ // eslint-disable-next-line max-len { name: 'description', content: `Crawlee helps you build and maintain your Python crawlers. It's open source and modern, with type hints for Python to help you catch bugs early.` }, // eslint-disable-next-line max-len { name: 'og:description', content: `Crawlee helps you build and maintain your Python crawlers. It's open source and modern, with type hints for Python to help you catch bugs early.` }, ], image: 'img/crawlee-python-og.png', footer: { links: [ { title: 'Docs', items: [ { label: 'Guides', to: 'docs/guides', }, { label: 'Examples', to: 'docs/examples', }, { label: 'API reference', to: 'api', }, { label: 'Changelog', to: 'docs/changelog', }, ], }, { title: 'Product', items: [ { label: 'Discord', href: 'https://discord.com/invite/jyEM2PRvMU', }, { label: 'Stack Overflow', href: 'https://stackoverflow.com/questions/tagged/crawlee-python', }, { label: 'Twitter', href: 'https://twitter.com/apify', }, { label: 'YouTube', href: 'https://www.youtube.com/apify', }, ], }, { title: 'More', items: [ { label: 'Apify platform', href: 'https://apify.com', }, { label: 'Docusaurus', href: 'https://docusaurus.io', }, { label: 'GitHub', href: 'https://github.com/apify/crawlee-python', }, ], }, ], }, algolia: { appId: '5JC94MPMLY', apiKey: '878493fcd7001e3c179b6db6796a999b', // search only (public) API key indexName: 'crawlee_python', placeholder: 'Search documentation', algoliaOptions: { facetFilters: ['version:VERSION'], }, translations: { button: { buttonText: 'Search documentation...', }, }, }, }), };