同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
@@ -0,0 +1,43 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler(
headless=False,
browser_type='chromium',
# Browser launch options
browser_launch_options={
# For support `msedge` channel you need to install it
# `playwright install msedge`
'channel': 'msedge',
'slow_mo': 200,
},
# Context launch options, applied to each page as it is created
browser_new_context_options={
'color_scheme': 'dark',
# Set headers
'extra_http_headers': {
'Custom-Header': 'my-header',
'Accept-Language': 'en',
},
# Set only User Agent
'user_agent': 'My-User-Agent',
},
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,48 @@
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING
from crawlee.browsers import BrowserPool
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
if TYPE_CHECKING:
from crawlee.browsers._browser_controller import BrowserController
from crawlee.browsers._browser_plugin import BrowserPlugin
logger = logging.getLogger(__name__)
async def main() -> None:
async with BrowserPool() as browser_pool:
@browser_pool.pre_launch_hook
async def log_browser_launch(page_id: str, plugin: BrowserPlugin) -> None:
"""Log before a new browser instance is launched."""
logger.info(f'Launching {plugin.browser_type} browser for page {page_id}...')
@browser_pool.post_launch_hook
async def log_browser_launched(
page_id: str, controller: BrowserController
) -> None:
"""Log after a new browser instance has been launched."""
logger.info(f'Browser launched for page {page_id}, controller: {controller}')
crawler = PlaywrightCrawler(
browser_pool=browser_pool,
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,77 @@
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING, Any
from crawlee.browsers import BrowserPool
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.storages import KeyValueStore
if TYPE_CHECKING:
from crawlee.browsers._browser_controller import BrowserController
from crawlee.browsers._types import CrawleePage
from crawlee.proxy_configuration import ProxyInfo
logger = logging.getLogger(__name__)
async def main() -> None:
async with BrowserPool() as browser_pool:
@browser_pool.pre_page_create_hook
async def log_page_init(
page_id: str,
_browser_controller: BrowserController,
_browser_new_context_options: dict[str, Any],
_proxy_info: ProxyInfo | None,
) -> None:
"""Log when a new page is about to be created."""
logger.info(f'Creating page {page_id}...')
@browser_pool.post_page_create_hook
async def set_viewport(
crawlee_page: CrawleePage, _browser_controller: BrowserController
) -> None:
"""Set a fixed viewport size on each newly created page."""
await crawlee_page.page.set_viewport_size({'width': 1280, 'height': 1024})
@browser_pool.pre_page_close_hook
async def save_screenshot(
crawlee_page: CrawleePage, _browser_controller: BrowserController
) -> None:
"""Save a screenshot to KeyValueStore before each page is closed."""
kvs = await KeyValueStore.open()
screenshot = await crawlee_page.page.screenshot()
await kvs.set_value(
key=f'screenshot-{crawlee_page.id}',
value=screenshot,
content_type='image/png',
)
logger.info(f'Saved screenshot for page {crawlee_page.id}.')
@browser_pool.post_page_close_hook
async def log_page_closed(
page_id: str, _browser_controller: BrowserController
) -> None:
"""Log after each page is closed."""
logger.info(f'Page {page_id} closed successfully.')
crawler = PlaywrightCrawler(
browser_pool=browser_pool,
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,38 @@
import asyncio
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
# Create a plugin for each required browser.
plugin_chromium = PlaywrightBrowserPlugin(
browser_type='chromium', max_open_pages_per_browser=1
)
plugin_firefox = PlaywrightBrowserPlugin(
browser_type='firefox', max_open_pages_per_browser=1
)
crawler = PlaywrightCrawler(
browser_pool=BrowserPool(plugins=[plugin_chromium, plugin_firefox]),
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
browser_name = (
context.page.context.browser.browser_type.name
if context.page.context.browser
else 'undefined'
)
context.log.info(f'Processing {context.request.url} with {browser_name} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev', 'https://apify.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import (
PlaywrightCrawler,
PlaywrightCrawlingContext,
PlaywrightPostNavCrawlingContext,
PlaywrightPreNavCrawlingContext,
)
from crawlee.errors import SessionError
async def main() -> None:
crawler = PlaywrightCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
@crawler.pre_navigation_hook
async def configure_page(context: PlaywrightPreNavCrawlingContext) -> None:
context.log.info(f'Navigating to {context.request.url} ...')
# block stylesheets, images, fonts and other static assets
# to speed up page loading
await context.block_requests()
@crawler.post_navigation_hook
async def custom_captcha_check(context: PlaywrightPostNavCrawlingContext) -> None:
# check if the page contains a captcha
captcha_element = context.page.locator('input[name="captcha"]').first
if await captcha_element.is_visible():
context.log.warning('Captcha detected! Skipping the page.')
raise SessionError('Captcha detected')
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,35 @@
import asyncio
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
from crawlee.crawlers import PlaywrightCrawler
async def main() -> None:
crawler = PlaywrightCrawler(
browser_pool=BrowserPool(
plugins=[
PlaywrightBrowserPlugin(
browser_type='chromium',
browser_launch_options={
'headless': False,
'channel': 'msedge',
'slow_mo': 200,
},
browser_new_context_options={
'color_scheme': 'dark',
'extra_http_headers': {
'Custom-Header': 'my-header',
'Accept-Language': 'en',
},
'user_agent': 'My-User-Agent',
},
)
]
)
)
# ...
if __name__ == '__main__':
asyncio.run(main())