48 lines
1.4 KiB
Python
48 lines
1.4 KiB
Python
import asyncio
|
|
|
|
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
|
from crawlee.storages import KeyValueStore
|
|
|
|
|
|
async def main() -> None:
|
|
crawler = PlaywrightCrawler(
|
|
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
|
max_requests_per_crawl=10,
|
|
# Headless mode, set to False to see the browser in action.
|
|
headless=False,
|
|
# Browser types supported by Playwright.
|
|
browser_type='chromium',
|
|
)
|
|
|
|
# Open the default key-value store.
|
|
kvs = await KeyValueStore.open()
|
|
|
|
# Define the default request handler, which will be called for every request.
|
|
@crawler.router.default_handler
|
|
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
|
context.log.info(f'Processing {context.request.url} ...')
|
|
|
|
# Capture the screenshot of the page using Playwright's API.
|
|
screenshot = await context.page.screenshot()
|
|
name = context.request.url.split('/')[-1]
|
|
|
|
# Store the screenshot in the key-value store.
|
|
await kvs.set_value(
|
|
key=f'screenshot-{name}',
|
|
value=screenshot,
|
|
content_type='image/png',
|
|
)
|
|
|
|
# Run the crawler with the initial list of URLs.
|
|
await crawler.run(
|
|
[
|
|
'https://crawlee.dev',
|
|
'https://apify.com',
|
|
'https://example.com',
|
|
]
|
|
)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main())
|