同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
headless=False,
|
||||
browser_type='chromium',
|
||||
# Browser launch options
|
||||
browser_launch_options={
|
||||
# For support `msedge` channel you need to install it
|
||||
# `playwright install msedge`
|
||||
'channel': 'msedge',
|
||||
'slow_mo': 200,
|
||||
},
|
||||
# Context launch options, applied to each page as it is created
|
||||
browser_new_context_options={
|
||||
'color_scheme': 'dark',
|
||||
# Set headers
|
||||
'extra_http_headers': {
|
||||
'Custom-Header': 'my-header',
|
||||
'Accept-Language': 'en',
|
||||
},
|
||||
# Set only User Agent
|
||||
'user_agent': 'My-User-Agent',
|
||||
},
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crawlee.browsers import BrowserPool
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crawlee.browsers._browser_controller import BrowserController
|
||||
from crawlee.browsers._browser_plugin import BrowserPlugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with BrowserPool() as browser_pool:
|
||||
|
||||
@browser_pool.pre_launch_hook
|
||||
async def log_browser_launch(page_id: str, plugin: BrowserPlugin) -> None:
|
||||
"""Log before a new browser instance is launched."""
|
||||
logger.info(f'Launching {plugin.browser_type} browser for page {page_id}...')
|
||||
|
||||
@browser_pool.post_launch_hook
|
||||
async def log_browser_launched(
|
||||
page_id: str, controller: BrowserController
|
||||
) -> None:
|
||||
"""Log after a new browser instance has been launched."""
|
||||
logger.info(f'Browser launched for page {page_id}, controller: {controller}')
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=browser_pool,
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from crawlee.browsers import BrowserPool
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storages import KeyValueStore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crawlee.browsers._browser_controller import BrowserController
|
||||
from crawlee.browsers._types import CrawleePage
|
||||
from crawlee.proxy_configuration import ProxyInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with BrowserPool() as browser_pool:
|
||||
|
||||
@browser_pool.pre_page_create_hook
|
||||
async def log_page_init(
|
||||
page_id: str,
|
||||
_browser_controller: BrowserController,
|
||||
_browser_new_context_options: dict[str, Any],
|
||||
_proxy_info: ProxyInfo | None,
|
||||
) -> None:
|
||||
"""Log when a new page is about to be created."""
|
||||
logger.info(f'Creating page {page_id}...')
|
||||
|
||||
@browser_pool.post_page_create_hook
|
||||
async def set_viewport(
|
||||
crawlee_page: CrawleePage, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Set a fixed viewport size on each newly created page."""
|
||||
await crawlee_page.page.set_viewport_size({'width': 1280, 'height': 1024})
|
||||
|
||||
@browser_pool.pre_page_close_hook
|
||||
async def save_screenshot(
|
||||
crawlee_page: CrawleePage, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Save a screenshot to KeyValueStore before each page is closed."""
|
||||
kvs = await KeyValueStore.open()
|
||||
|
||||
screenshot = await crawlee_page.page.screenshot()
|
||||
await kvs.set_value(
|
||||
key=f'screenshot-{crawlee_page.id}',
|
||||
value=screenshot,
|
||||
content_type='image/png',
|
||||
)
|
||||
logger.info(f'Saved screenshot for page {crawlee_page.id}.')
|
||||
|
||||
@browser_pool.post_page_close_hook
|
||||
async def log_page_closed(
|
||||
page_id: str, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Log after each page is closed."""
|
||||
logger.info(f'Page {page_id} closed successfully.')
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=browser_pool,
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a plugin for each required browser.
|
||||
plugin_chromium = PlaywrightBrowserPlugin(
|
||||
browser_type='chromium', max_open_pages_per_browser=1
|
||||
)
|
||||
plugin_firefox = PlaywrightBrowserPlugin(
|
||||
browser_type='firefox', max_open_pages_per_browser=1
|
||||
)
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=BrowserPool(plugins=[plugin_chromium, plugin_firefox]),
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
browser_name = (
|
||||
context.page.context.browser.browser_type.name
|
||||
if context.page.context.browser
|
||||
else 'undefined'
|
||||
)
|
||||
context.log.info(f'Processing {context.request.url} with {browser_name} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev', 'https://apify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPostNavCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
from crawlee.errors import SessionError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def configure_page(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
context.log.info(f'Navigating to {context.request.url} ...')
|
||||
|
||||
# block stylesheets, images, fonts and other static assets
|
||||
# to speed up page loading
|
||||
await context.block_requests()
|
||||
|
||||
@crawler.post_navigation_hook
|
||||
async def custom_captcha_check(context: PlaywrightPostNavCrawlingContext) -> None:
|
||||
# check if the page contains a captcha
|
||||
captcha_element = context.page.locator('input[name="captcha"]').first
|
||||
if await captcha_element.is_visible():
|
||||
context.log.warning('Captcha detected! Skipping the page.')
|
||||
raise SessionError('Captcha detected')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=BrowserPool(
|
||||
plugins=[
|
||||
PlaywrightBrowserPlugin(
|
||||
browser_type='chromium',
|
||||
browser_launch_options={
|
||||
'headless': False,
|
||||
'channel': 'msedge',
|
||||
'slow_mo': 200,
|
||||
},
|
||||
browser_new_context_options={
|
||||
'color_scheme': 'dark',
|
||||
'extra_http_headers': {
|
||||
'Custom-Header': 'my-header',
|
||||
'Accept-Language': 'en',
|
||||
},
|
||||
'user_agent': 'My-User-Agent',
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user