同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
@@ -0,0 +1,20 @@
import asyncio
from crawlee.fingerprint_suite import (
DefaultFingerprintGenerator,
HeaderGeneratorOptions,
ScreenOptions,
)
async def main() -> None:
fingerprint_generator = DefaultFingerprintGenerator(
header_options=HeaderGeneratorOptions(browsers=['chrome']),
screen_options=ScreenOptions(min_width=400),
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,81 @@
import asyncio
# CloakBrowser is an external package. Install it separately.
from cloakbrowser.config import IGNORE_DEFAULT_ARGS, get_default_stealth_args
from cloakbrowser.download import ensure_binary
from typing_extensions import override
from crawlee.browsers import (
BrowserPool,
PlaywrightBrowserController,
PlaywrightBrowserPlugin,
)
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
class CloakBrowserPlugin(PlaywrightBrowserPlugin):
"""Example browser plugin that uses CloakBrowser's patched Chromium,
but otherwise keeps the functionality of PlaywrightBrowserPlugin.
"""
@override
async def new_browser(self) -> PlaywrightBrowserController:
if not self._playwright:
raise RuntimeError('Playwright browser plugin is not initialized.')
binary_path = ensure_binary()
stealth_args = get_default_stealth_args()
# Merge CloakBrowser stealth args with any user-provided launch options.
launch_options = dict(self._browser_launch_options)
launch_options.pop('executable_path', None)
launch_options.pop('chromium_sandbox', None)
existing_args = list(launch_options.pop('args', []))
launch_options['args'] = [*existing_args, *stealth_args]
return PlaywrightBrowserController(
browser=await self._playwright.chromium.launch(
executable_path=binary_path,
ignore_default_args=IGNORE_DEFAULT_ARGS,
**launch_options,
),
max_open_pages_per_browser=1,
# CloakBrowser handles fingerprints at the binary level.
header_generator=None,
)
async def main() -> None:
crawler = PlaywrightCrawler(
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
# Custom browser pool. Gives users full control over browsers used by the crawler.
browser_pool=BrowserPool(plugins=[CloakBrowserPlugin()]),
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Extract some data from the page using Playwright's API.
posts = await context.page.query_selector_all('.athing')
for post in posts:
# Get the HTML elements for the title and rank within each post.
title_element = await post.query_selector('.title a')
# Extract the data we want from the elements.
title = await title_element.inner_text() if title_element else None
# Push the extracted data to the default dataset.
await context.push_data({'title': title})
# Find a link to the next page and enqueue it if it exists.
await context.enqueue_links(selector='.morelink')
# Run the crawler with the initial list of URLs.
await crawler.run(['https://news.ycombinator.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,23 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
# Fingerprint generator is used by default.
crawler = PlaywrightCrawler()
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Find a link to the next page and enqueue it if it exists.
await context.enqueue_links(selector='.morelink')
# Run the crawler with the initial list of URLs.
await crawler.run(['https://news.ycombinator.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,61 @@
import asyncio
import io
from pathlib import Path
from warcio.statusandheaders import StatusAndHeaders
from warcio.warcwriter import WARCWriter
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
async def archive_response(context: ParselCrawlingContext, writer: WARCWriter) -> None:
"""Helper function for archiving response in WARC format."""
# Create WARC records for response
response_body = await context.http_response.read()
response_payload_stream = io.BytesIO(response_body)
response_headers = StatusAndHeaders(
str(context.http_response.status_code),
context.http_response.headers,
protocol='HTTP/1.1',
)
response_record = writer.create_warc_record(
context.request.url,
'response',
payload=response_payload_stream,
length=len(response_body),
http_headers=response_headers,
)
writer.write_record(response_record)
async def main() -> None:
crawler = ParselCrawler(
max_requests_per_crawl=10,
)
# Create a WARC archive file a prepare the writer.
archive = Path('example.warc.gz')
with archive.open('wb') as output:
writer = WARCWriter(output, gzip=True)
# Create a WARC info record to store metadata about the archive.
warcinfo_payload = {
'software': 'Crawlee',
'format': 'WARC/1.1',
'description': 'Example archive created with ParselCrawler',
}
writer.write_record(writer.create_warcinfo_record(archive.name, warcinfo_payload))
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Archiving {context.request.url} ...')
await archive_response(context=context, writer=writer)
await context.enqueue_links(strategy='same-domain')
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,85 @@
import asyncio
import io
import logging
from functools import partial
from pathlib import Path
from playwright.async_api import Request
from warcio.statusandheaders import StatusAndHeaders
from warcio.warcwriter import WARCWriter
from crawlee.crawlers import (
PlaywrightCrawler,
PlaywrightCrawlingContext,
PlaywrightPreNavCrawlingContext,
)
async def archive_response(
request: Request, writer: WARCWriter, logger: logging.Logger
) -> None:
"""Helper function for archiving response in WARC format."""
response = await request.response()
if not response:
logger.warning(f'Could not get response {request.url}')
return
try:
response_body = await response.body()
except Exception as e:
logger.warning(f'Could not get response body for {response.url}: {e}')
return
logger.info(f'Archiving resource {response.url}')
response_payload_stream = io.BytesIO(response_body)
response_headers = StatusAndHeaders(
str(response.status), response.headers, protocol='HTTP/1.1'
)
response_record = writer.create_warc_record(
response.url,
'response',
payload=response_payload_stream,
length=len(response_body),
http_headers=response_headers,
)
writer.write_record(response_record)
async def main() -> None:
crawler = PlaywrightCrawler(
max_requests_per_crawl=1,
headless=False,
)
# Create a WARC archive file a prepare the writer.
archive = Path('example.warc.gz')
with archive.open('wb') as output:
writer = WARCWriter(output, gzip=True)
# Create a WARC info record to store metadata about the archive.
warcinfo_payload = {
'software': 'Crawlee',
'format': 'WARC/1.1',
'description': 'Example archive created with PlaywrightCrawler',
}
writer.write_record(writer.create_warcinfo_record(archive.name, warcinfo_payload))
@crawler.pre_navigation_hook
async def archiving_hook(context: PlaywrightPreNavCrawlingContext) -> None:
# Ensure that all responses with additional resources are archived
context.page.on(
'requestfinished',
partial(archive_response, logger=context.log, writer=writer),
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
# For some sites, where the content loads dynamically,
# it is needed to scroll the page to load all content.
# It slows down the crawling, but ensures that all content is loaded.
await context.infinite_scroll()
await context.enqueue_links(strategy='same-domain')
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,30 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
crawler = PlaywrightCrawler(
# Use the local wayback server as a proxy
proxy_configuration=ProxyConfiguration(proxy_urls=['http://localhost:8080/']),
# Ignore the HTTPS errors if you have not followed pywb CA setup instructions
browser_launch_options={'ignore_https_errors': True},
max_requests_per_crawl=10,
headless=False,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Archiving {context.request.url} ...')
# For some sites, where the content loads dynamically,
# it is needed to scroll the page to load all content.
# It slows down the crawling, but ensures that all content is loaded.
await context.infinite_scroll()
await context.enqueue_links(strategy='same-domain')
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,45 @@
import asyncio
import json
from crawlee import HttpHeaders
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
from crawlee.errors import HttpStatusCodeError
from crawlee.sessions import SessionPool
# Using a placeholder refresh token for this example
REFRESH_TOKEN = 'PLACEHOLDER'
UNAUTHORIZED_CODE = 401
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=2,
# Only treat 403 as a blocking status code, not 401
session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}),
# Don't treat 401 responses as errors
ignore_http_error_status_codes=[UNAUTHORIZED_CODE],
)
@crawler.router.default_handler
async def default_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Now we can handle 401 responses ourselves
if context.http_response.status_code == UNAUTHORIZED_CODE:
# Get a fresh access token
headers = {'authorization': f'Bearer {REFRESH_TOKEN}'}
response = await context.send_request(
'https://placeholder.org/refresh', headers=headers
)
data = json.loads(await response.read())
# Add the new token to our `Request` headers
context.request.headers |= HttpHeaders(
{'authorization': f'Bearer {data["access_token"]}'},
)
# Trigger a retry with our updated headers
raise HttpStatusCodeError('Unauthorized', status_code=UNAUTHORIZED_CODE)
await crawler.run(['http://httpbingo.org/status/401'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,30 @@
import asyncio
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import HttpStatusCodeError, SessionError
async def main() -> None:
crawler = HttpCrawler(max_request_retries=5)
# Create a parsing error for demonstration
@crawler.router.default_handler
async def default_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
raise ValueError('Simulated parsing error')
# This handler runs before any retry attempts
@crawler.error_handler
async def retry_handler(context: BasicCrawlingContext, error: Exception) -> None:
context.log.error(f'Failed request {context.request.url}')
# Only allow retries for network-related errors
if not isinstance(error, (SessionError, HttpStatusCodeError)):
context.log.error('Non-network error detected')
# Stop further retry attempts for this `Request`
context.request.no_retry = True
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,40 @@
import asyncio
from crawlee import Request
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import ProxyError
async def main() -> None:
# Set how many session rotations will happen before calling the error handler
# when ProxyError occurs
crawler = HttpCrawler(max_session_rotations=5, max_request_retries=6)
# For this example, we'll create a proxy error in our handler
@crawler.router.default_handler
async def default_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
raise ProxyError('Simulated proxy error')
# This handler runs after all retry attempts are exhausted
@crawler.failed_request_handler
async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
context.log.error(f'Failed request {context.request.url}, after 5 rotations')
request = context.request
# For proxy errors, we can add a new `Request` to try again
if isinstance(error, ProxyError) and not request.unique_key.startswith('retry'):
context.log.info(f'Retrying {request.url} ...')
# Create a new `Request` with a modified key to avoid deduplication
new_request = Request.from_url(
request.url, unique_key=f'retry{request.unique_key}'
)
# Add the new `Request` to the `Queue`
rq = await crawler.get_request_manager()
await rq.add_request(new_request)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import CurlImpersonateHttpClient
async def main() -> None:
http_client = CurlImpersonateHttpClient(
# Optional additional keyword arguments for `curl_cffi.requests.AsyncSession`.
timeout=10,
impersonate='chrome131',
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import HttpxHttpClient
async def main() -> None:
http_client = HttpxHttpClient(
# Optional additional keyword arguments for `httpx.AsyncClient`.
timeout=10,
follow_redirects=True,
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,43 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import ImpitHttpClient
async def main() -> None:
http_client = ImpitHttpClient(
# Optional additional keyword arguments for `impit.AsyncClient`.
http3=True,
browser='firefox',
verify=True,
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,35 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
# Create a BeautifulSoupCrawler instance
crawler = BeautifulSoupCrawler(
# Limit the crawl to 10 requests
max_requests_per_crawl=10,
)
# Define the default request handler
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract data using BeautifulSoup
data = {
'url': context.request.url,
'title': context.soup.title.string if context.soup.title else None,
}
# Push extracted data to the dataset
await context.push_data(data)
# Enqueue links found on the page for further crawling
await context.enqueue_links()
# Run the crawler
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,52 @@
import asyncio
import re
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
# Create an HttpCrawler instance - no automatic parsing
crawler = HttpCrawler(
# Limit the crawl to 10 requests
max_requests_per_crawl=10,
)
# Define the default request handler
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Get the raw response content
response_body = await context.http_response.read()
response_text = response_body.decode('utf-8')
# Extract title manually using regex (since we don't have a parser)
title_match = re.search(
r'<title[^>]*>([^<]+)</title>', response_text, re.IGNORECASE
)
title = title_match.group(1).strip() if title_match else None
# Extract basic information
data = {
'url': context.request.url,
'title': title,
}
# Push extracted data to the dataset
await context.push_data(data)
# Simple link extraction for further crawling
href_pattern = r'href=["\']([^"\']+)["\']'
matches = re.findall(href_pattern, response_text, re.IGNORECASE)
# Enqueue first few links found (limit to avoid too many requests)
for href in matches[:3]:
if href.startswith('http') and 'crawlee.dev' in href:
await context.add_requests([href])
# Run the crawler
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,63 @@
import asyncio
from pydantic import ValidationError
from selectolax.lexbor import LexborHTMLParser
from yarl import URL
from crawlee import Request
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=1,
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Parse the HTML content using Selectolax with Lexbor backend.
parsed_html = LexborHTMLParser(await context.http_response.read())
# Extract data from the page.
data = {
'url': context.request.url,
'title': parsed_html.css_first('title').text(),
'h1s': [h1.text() for h1 in parsed_html.css('h1')],
'h2s': [h2.text() for h2 in parsed_html.css('h2')],
'h3s': [h3.text() for h3 in parsed_html.css('h3')],
}
await context.push_data(data)
# Css selector to extract valid href attributes.
links_selector = (
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
)
base_url = URL(context.request.url)
extracted_requests = []
# Extract links.
for item in parsed_html.css(links_selector):
href = item.attributes.get('href')
if not href:
continue
# Convert relative URLs to absolute if needed.
url = str(base_url.join(URL(href)))
try:
request = Request.from_url(url)
except ValidationError as exc:
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
continue
extracted_requests.append(request)
# Add extracted requests to the queue with the same-domain strategy.
await context.add_requests(extracted_requests, strategy='same-domain')
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,61 @@
import asyncio
from lxml import html
from pydantic import ValidationError
from crawlee import Request
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=1,
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Parse the HTML content using lxml.
parsed_html = html.fromstring(await context.http_response.read())
# Extract data from the page.
data = {
'url': context.request.url,
'title': parsed_html.findtext('.//title'),
'h1s': [h1.text_content() for h1 in parsed_html.findall('.//h1')],
'h2s': [h2.text_content() for h2 in parsed_html.findall('.//h2')],
'h3s': [h3.text_content() for h3 in parsed_html.findall('.//h3')],
}
await context.push_data(data)
# Convert relative URLs to absolute before extracting links.
parsed_html.make_links_absolute(context.request.url, resolve_base_href=True)
# Xpath 1.0 selector for extracting valid href attributes.
links_xpath = (
'//a/@href[not(starts-with(., "#")) '
'and not(starts-with(., "javascript:")) '
'and not(starts-with(., "mailto:"))]'
)
extracted_requests = []
# Extract links.
for url in parsed_html.xpath(links_xpath):
try:
request = Request.from_url(url)
except ValidationError as exc:
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
continue
extracted_requests.append(request)
# Add extracted requests to the queue with the same-domain strategy.
await context.add_requests(extracted_requests, strategy='same-domain')
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,77 @@
import asyncio
from lxml import html
from pydantic import ValidationError
from saxonche import PySaxonProcessor
from crawlee import Request
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=1,
max_requests_per_crawl=10,
)
# Create Saxon processor once and reuse across requests.
saxon_proc = PySaxonProcessor(license=False)
xpath_proc = saxon_proc.new_xpath_processor()
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Parse HTML with lxml.
parsed_html = html.fromstring(await context.http_response.read())
# Convert relative URLs to absolute before extracting links.
parsed_html.make_links_absolute(context.request.url, resolve_base_href=True)
# Convert parsed HTML to XML for Saxon processing.
xml = html.tostring(parsed_html, encoding='unicode', method='xml')
# Parse XML with Saxon.
parsed_xml = saxon_proc.parse_xml(xml_text=xml)
# Set the parsed context for XPath evaluation.
xpath_proc.set_context(xdm_item=parsed_xml)
# Extract data using XPath 2.0 string() function.
data = {
'url': context.request.url,
'title': xpath_proc.evaluate_single('.//title/string()'),
'h1s': [str(h) for h in (xpath_proc.evaluate('//h1/string()') or [])],
'h2s': [str(h) for h in (xpath_proc.evaluate('//h2/string()') or [])],
'h3s': [str(h) for h in (xpath_proc.evaluate('//h3/string()') or [])],
}
await context.push_data(data)
# XPath 2.0 with distinct-values() to get unique links and remove fragments.
links_xpath = """
distinct-values(
for $href in //a/@href[
not(starts-with(., "#"))
and not(starts-with(., "javascript:"))
and not(starts-with(., "mailto:"))
]
return replace($href, "#.*$", "")
)
"""
extracted_requests = []
# Extract links.
for item in xpath_proc.evaluate(links_xpath) or []:
url = item.string_value
try:
request = Request.from_url(url)
except ValidationError as exc:
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
continue
extracted_requests.append(request)
# Add extracted requests to the queue with the same-domain strategy.
await context.add_requests(extracted_requests, strategy='same-domain')
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,35 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
async def main() -> None:
# Create a ParselCrawler instance
crawler = ParselCrawler(
# Limit the crawl to 10 requests
max_requests_per_crawl=10,
)
# Define the default request handler
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract data using Parsel's XPath and CSS selectors
data = {
'url': context.request.url,
'title': context.selector.xpath('//title/text()').get(),
}
# Push extracted data to the dataset
await context.push_data(data)
# Enqueue links found on the page for further crawling
await context.enqueue_links()
# Run the crawler
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,64 @@
import asyncio
from pydantic import ValidationError
from pyquery import PyQuery
from yarl import URL
from crawlee import Request
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=1,
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Parse the HTML content using PyQuery.
parsed_html = PyQuery(await context.http_response.read())
# Extract data using jQuery-style selectors.
data = {
'url': context.request.url,
'title': parsed_html('title').text(),
'h1s': [h1.text() for h1 in parsed_html('h1').items()],
'h2s': [h2.text() for h2 in parsed_html('h2').items()],
'h3s': [h3.text() for h3 in parsed_html('h3').items()],
}
await context.push_data(data)
# Css selector to extract valid href attributes.
links_selector = (
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
)
base_url = URL(context.request.url)
extracted_requests = []
# Extract links.
for item in parsed_html(links_selector).items():
href = item.attr('href')
if not href:
continue
# Convert relative URLs to absolute if needed.
url = str(base_url.join(URL(str(href))))
try:
request = Request.from_url(url)
except ValidationError as exc:
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
continue
extracted_requests.append(request)
# Add extracted requests to the queue with the same-domain strategy.
await context.add_requests(extracted_requests, strategy='same-domain')
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,74 @@
import asyncio
from pydantic import ValidationError
from scrapling.parser import Selector
from yarl import URL
from crawlee import Request
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
async def main() -> None:
crawler = HttpCrawler(
max_request_retries=1,
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Parse the HTML content using Scrapling.
page = Selector(await context.http_response.read(), url=context.request.url)
# Extract data using Xpath selectors with .get_all_text method for full text
# content.
title_el = page.xpath_first('//title')
data = {
'url': context.request.url,
'title': title_el.text if isinstance(title_el, Selector) else title_el,
'h1s': [
h1.get_all_text() if isinstance(h1, Selector) else h1
for h1 in page.xpath('//h1')
],
'h2s': [
h2.get_all_text() if isinstance(h2, Selector) else h2
for h2 in page.xpath('//h2')
],
'h3s': [
h3.get_all_text() if isinstance(h3, Selector) else h3
for h3 in page.xpath('//h3')
],
}
await context.push_data(data)
# Css selector to extract valid href attributes.
links_selector = (
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
)
base_url = URL(context.request.url)
extracted_requests = []
# Extract links.
for item in page.css(links_selector):
href = item.attrib.get('href') if isinstance(item, Selector) else None
if not href:
continue
# Convert relative URLs to absolute if needed.
url = str(base_url.join(URL(href)))
try:
request = Request.from_url(url)
except ValidationError as exc:
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
continue
extracted_requests.append(request)
# Add extracted requests to the queue with the same-domain strategy.
await context.add_requests(extracted_requests, strategy='same-domain')
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,34 @@
import asyncio
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
AdaptivePlaywrightCrawlingContext,
)
from .selectolax_parser import SelectolaxLexborParser
async def main() -> None:
crawler: AdaptivePlaywrightCrawler = AdaptivePlaywrightCrawler(
max_requests_per_crawl=10,
# Use custom Selectolax parser for static content parsing.
static_parser=SelectolaxLexborParser(),
)
@crawler.router.default_handler
async def handle_request(context: AdaptivePlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
data = {
'url': context.request.url,
'title': await context.query_selector_one('title'),
}
await context.push_data(data)
await context.enqueue_links()
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,35 @@
from dataclasses import dataclass, fields
from selectolax.lexbor import LexborHTMLParser
from typing_extensions import Self
from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext
# Custom context for Selectolax parser, you can add your own methods here
# to facilitate working with the parsed document.
@dataclass(frozen=True)
class SelectolaxLexborContext(ParsedHttpCrawlingContext[LexborHTMLParser]):
"""Crawling context providing access to the parsed page.
This context is passed to request handlers and includes all standard
context methods (push_data, enqueue_links, etc.) plus custom helpers.
"""
@property
def parser(self) -> LexborHTMLParser:
"""Convenient alias for accessing the parsed document."""
return self.parsed_content
@classmethod
def from_parsed_http_crawling_context(
cls, context: ParsedHttpCrawlingContext[LexborHTMLParser]
) -> Self:
"""Create custom context from the base context.
Copies all fields from the base context to preserve framework
functionality while adding custom interface.
"""
return cls(
**{field.name: getattr(context, field.name) for field in fields(context)}
)
@@ -0,0 +1,47 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from selectolax.lexbor import LexborHTMLParser, LexborNode
from crawlee.crawlers import AbstractHttpCrawler, HttpCrawlerOptions
from .selectolax_context import SelectolaxLexborContext
from .selectolax_parser import SelectolaxLexborParser
if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from typing_extensions import Unpack
from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext
# Custom crawler using custom context, It is optional and you can use
# AbstractHttpCrawler directly with SelectolaxLexborParser if you don't need
# any custom context methods.
class SelectolaxLexborCrawler(
AbstractHttpCrawler[SelectolaxLexborContext, LexborHTMLParser, LexborNode]
):
"""Custom crawler using Selectolax Lexbor for HTML parsing."""
def __init__(
self,
**kwargs: Unpack[HttpCrawlerOptions[SelectolaxLexborContext]],
) -> None:
# Final step converts the base context to custom context type.
async def final_step(
context: ParsedHttpCrawlingContext[LexborHTMLParser],
) -> AsyncGenerator[SelectolaxLexborContext, None]:
# Yield custom context wrapping with additional functionality around the base
# context.
yield SelectolaxLexborContext.from_parsed_http_crawling_context(context)
# Build context pipeline: HTTP request -> parsing -> custom context.
kwargs['_context_pipeline'] = (
self._create_static_content_crawler_pipeline().compose(final_step)
)
super().__init__(
parser=SelectolaxLexborParser(),
**kwargs,
)
@@ -0,0 +1,27 @@
import asyncio
from .selectolax_crawler import SelectolaxLexborContext, SelectolaxLexborCrawler
async def main() -> None:
crawler = SelectolaxLexborCrawler(
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def handle_request(context: SelectolaxLexborContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
data = {
'url': context.request.url,
'title': context.parser.css_first('title').text(),
}
await context.push_data(data)
await context.enqueue_links()
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,60 @@
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
from selectolax.lexbor import LexborHTMLParser, LexborNode
from typing_extensions import override
from crawlee.crawlers._abstract_http import AbstractHttpParser
if TYPE_CHECKING:
from collections.abc import Iterable, Sequence
from crawlee.http_clients import HttpResponse
class SelectolaxLexborParser(AbstractHttpParser[LexborHTMLParser, LexborNode]):
"""Parser for parsing HTTP response using Selectolax Lexbor."""
@override
async def parse(self, response: HttpResponse) -> LexborHTMLParser:
"""Parse HTTP response body into a document object."""
response_body = await response.read()
# Run parsing in a thread to avoid blocking the event loop.
return await asyncio.to_thread(LexborHTMLParser, response_body)
@override
async def parse_text(self, text: str) -> LexborHTMLParser:
"""Parse raw HTML string into a document object."""
return LexborHTMLParser(text)
@override
async def select(
self, parsed_content: LexborHTMLParser, selector: str
) -> Sequence[LexborNode]:
"""Select elements matching a CSS selector."""
return tuple(item for item in parsed_content.css(selector))
@override
def is_matching_selector(
self, parsed_content: LexborHTMLParser, selector: str
) -> bool:
"""Check if any element matches the selector."""
return parsed_content.css_first(selector) is not None
@override
def find_links(
self, parsed_content: LexborHTMLParser, selector: str, attribute: str
) -> Iterable[str]:
"""Extract href attributes from elements matching the selector.
Used by `enqueue_links` helper to discover URLs.
"""
link: LexborNode
urls: list[str] = []
for link in parsed_content.css(selector):
url = link.attributes.get(attribute)
if url:
urls.append(url.strip())
return urls
@@ -0,0 +1,85 @@
import asyncio
import json
from datetime import datetime, timedelta
from crawlee import ConcurrencySettings, Request
from crawlee.crawlers import (
HttpCrawler,
HttpCrawlingContext,
)
from crawlee.sessions import SessionPool
async def main() -> None:
crawler = HttpCrawler(
max_requests_per_crawl=10,
# Configure to use a single persistent session throughout the crawl
max_session_rotations=0,
# Limit request rate to avoid triggering anti-scraping measures
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
session_pool=SessionPool(
max_pool_size=1,
create_session_settings={
# Set high value to ensure the session isn't replaced during crawling
'max_usage_count': 999_999,
# Set high value to prevent session expiration during crawling
'max_age': timedelta(hours=999_999),
# Higher error tolerance before the session is considered blocked
# Make sure you implement proper error handling in your code
'max_error_score': 100,
},
),
)
# Default request handler for normal page processing
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Specialized handler for the login API request
@crawler.router.handler('login')
async def login_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing login at {context.request.url} ...')
# Verify that a session is available before proceeding
if not context.session:
raise RuntimeError('Session not found')
# Parse the API response containing authentication tokens and user data
data = json.loads(await context.http_response.read())
# Extract authentication data from the response
token = data['token']
expires = data['expires'].replace('Z', '+00:00')
expires_int = int(datetime.fromisoformat(expires).timestamp())
user_id = data['userId']
username = data['username']
# Set authentication cookies in the session that will be used
# for subsequent requests
context.session.cookies.set(name='token', value=token, expires=expires_int)
context.session.cookies.set(name='userID', value=user_id)
context.session.cookies.set(name='userName', value=username)
# After successful authentication, continue crawling with the
# authenticated session
await context.add_requests(['https://demoqa.com/BookStore/v1/Books'])
# Create a POST request to the authentication API endpoint
# This will trigger the login_handler when executed
request = Request.from_url(
'https://demoqa.com/Account/v1/Login',
label='login',
method='POST',
payload=json.dumps(
{'userName': 'crawlee_test', 'password': 'Test1234!'}
).encode(),
headers={'Content-Type': 'application/json'},
)
# Start the crawling process with the login request
await crawler.run([request])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,70 @@
import asyncio
from datetime import timedelta
from crawlee import ConcurrencySettings, Request
from crawlee.crawlers import (
PlaywrightCrawler,
PlaywrightCrawlingContext,
)
from crawlee.sessions import SessionPool
async def main() -> None:
crawler = PlaywrightCrawler(
max_requests_per_crawl=10,
headless=True,
browser_type='chromium',
# We only have one session and it shouldn't rotate
max_session_rotations=0,
# Limit crawling intensity to avoid blocking
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
session_pool=SessionPool(
# Limit the pool to one session
max_pool_size=1,
create_session_settings={
# High value for session usage limit
'max_usage_count': 999_999,
# High value for session lifetime
'max_age': timedelta(hours=999_999),
# High score allows the session to encounter more errors
# before crawlee decides the session is blocked
# Make sure you know how to handle these errors
'max_error_score': 100,
},
),
)
# The main handler for processing requests
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# A handler for the login page
@crawler.router.handler('login')
async def login_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing login {context.request.url} ...')
# Check if the session is available
if not context.session:
raise RuntimeError('Session not found')
# Entering data into the form, `delay` to simulate human typing
# Without this, the data will be entered instantly
await context.page.type('#userName', 'crawlee_test', delay=100)
await context.page.type('#password', 'Test1234!', delay=100)
await context.page.click('#login', delay=100)
# Wait for an element confirming that we have successfully
# logged in to the site
await context.page.locator('#userName-value').first.wait_for(state='visible')
context.log.info('Login successful!')
# Moving on to the basic flow of crawling
await context.add_requests(['https://demoqa.com/books'])
# We start crawling with login. This is necessary to access the rest of the pages
await crawler.run([Request.from_url('https://demoqa.com/login', label='login')])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,43 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler(
headless=False,
browser_type='chromium',
# Browser launch options
browser_launch_options={
# For support `msedge` channel you need to install it
# `playwright install msedge`
'channel': 'msedge',
'slow_mo': 200,
},
# Context launch options, applied to each page as it is created
browser_new_context_options={
'color_scheme': 'dark',
# Set headers
'extra_http_headers': {
'Custom-Header': 'my-header',
'Accept-Language': 'en',
},
# Set only User Agent
'user_agent': 'My-User-Agent',
},
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,48 @@
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING
from crawlee.browsers import BrowserPool
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
if TYPE_CHECKING:
from crawlee.browsers._browser_controller import BrowserController
from crawlee.browsers._browser_plugin import BrowserPlugin
logger = logging.getLogger(__name__)
async def main() -> None:
async with BrowserPool() as browser_pool:
@browser_pool.pre_launch_hook
async def log_browser_launch(page_id: str, plugin: BrowserPlugin) -> None:
"""Log before a new browser instance is launched."""
logger.info(f'Launching {plugin.browser_type} browser for page {page_id}...')
@browser_pool.post_launch_hook
async def log_browser_launched(
page_id: str, controller: BrowserController
) -> None:
"""Log after a new browser instance has been launched."""
logger.info(f'Browser launched for page {page_id}, controller: {controller}')
crawler = PlaywrightCrawler(
browser_pool=browser_pool,
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,77 @@
from __future__ import annotations
import asyncio
import logging
from typing import TYPE_CHECKING, Any
from crawlee.browsers import BrowserPool
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.storages import KeyValueStore
if TYPE_CHECKING:
from crawlee.browsers._browser_controller import BrowserController
from crawlee.browsers._types import CrawleePage
from crawlee.proxy_configuration import ProxyInfo
logger = logging.getLogger(__name__)
async def main() -> None:
async with BrowserPool() as browser_pool:
@browser_pool.pre_page_create_hook
async def log_page_init(
page_id: str,
_browser_controller: BrowserController,
_browser_new_context_options: dict[str, Any],
_proxy_info: ProxyInfo | None,
) -> None:
"""Log when a new page is about to be created."""
logger.info(f'Creating page {page_id}...')
@browser_pool.post_page_create_hook
async def set_viewport(
crawlee_page: CrawleePage, _browser_controller: BrowserController
) -> None:
"""Set a fixed viewport size on each newly created page."""
await crawlee_page.page.set_viewport_size({'width': 1280, 'height': 1024})
@browser_pool.pre_page_close_hook
async def save_screenshot(
crawlee_page: CrawleePage, _browser_controller: BrowserController
) -> None:
"""Save a screenshot to KeyValueStore before each page is closed."""
kvs = await KeyValueStore.open()
screenshot = await crawlee_page.page.screenshot()
await kvs.set_value(
key=f'screenshot-{crawlee_page.id}',
value=screenshot,
content_type='image/png',
)
logger.info(f'Saved screenshot for page {crawlee_page.id}.')
@browser_pool.post_page_close_hook
async def log_page_closed(
page_id: str, _browser_controller: BrowserController
) -> None:
"""Log after each page is closed."""
logger.info(f'Page {page_id} closed successfully.')
crawler = PlaywrightCrawler(
browser_pool=browser_pool,
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,38 @@
import asyncio
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
# Create a plugin for each required browser.
plugin_chromium = PlaywrightBrowserPlugin(
browser_type='chromium', max_open_pages_per_browser=1
)
plugin_firefox = PlaywrightBrowserPlugin(
browser_type='firefox', max_open_pages_per_browser=1
)
crawler = PlaywrightCrawler(
browser_pool=BrowserPool(plugins=[plugin_chromium, plugin_firefox]),
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
browser_name = (
context.page.context.browser.browser_type.name
if context.page.context.browser
else 'undefined'
)
context.log.info(f'Processing {context.request.url} with {browser_name} ...')
await context.enqueue_links()
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev', 'https://apify.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import (
PlaywrightCrawler,
PlaywrightCrawlingContext,
PlaywrightPostNavCrawlingContext,
PlaywrightPreNavCrawlingContext,
)
from crawlee.errors import SessionError
async def main() -> None:
crawler = PlaywrightCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
await context.enqueue_links()
@crawler.pre_navigation_hook
async def configure_page(context: PlaywrightPreNavCrawlingContext) -> None:
context.log.info(f'Navigating to {context.request.url} ...')
# block stylesheets, images, fonts and other static assets
# to speed up page loading
await context.block_requests()
@crawler.post_navigation_hook
async def custom_captcha_check(context: PlaywrightPostNavCrawlingContext) -> None:
# check if the page contains a captcha
captcha_element = context.page.locator('input[name="captcha"]').first
if await captcha_element.is_visible():
context.log.warning('Captcha detected! Skipping the page.')
raise SessionError('Captcha detected')
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,35 @@
import asyncio
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
from crawlee.crawlers import PlaywrightCrawler
async def main() -> None:
crawler = PlaywrightCrawler(
browser_pool=BrowserPool(
plugins=[
PlaywrightBrowserPlugin(
browser_type='chromium',
browser_launch_options={
'headless': False,
'channel': 'msedge',
'slow_mo': 200,
},
browser_new_context_options={
'color_scheme': 'dark',
'extra_http_headers': {
'Custom-Header': 'my-header',
'Accept-Language': 'en',
},
'user_agent': 'My-User-Agent',
},
)
]
)
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
from datetime import timedelta
from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
@crawler.router.default_handler
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
# Locate element h2 within 5 seconds
h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
# Do stuff with element found by the selector
context.log.info(h2)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
from crawlee.crawlers import AdaptivePlaywrightCrawler
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
# Arguments relevant only for PlaywrightCrawler
playwright_crawler_specific_kwargs={
'headless': False,
'browser_type': 'chromium',
},
# Common arguments relevant to all crawlers
max_crawl_depth=5,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
from crawlee.crawlers import AdaptivePlaywrightCrawler
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
# Arguments relevant only for PlaywrightCrawler
playwright_crawler_specific_kwargs={
'headless': False,
'browser_type': 'chromium',
},
# Common arguments relevant to all crawlers
max_crawl_depth=5,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,72 @@
import asyncio
from crawlee import Request
from crawlee._types import RequestHandlerRunResult
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
RenderingType,
RenderingTypePrediction,
RenderingTypePredictor,
)
class CustomRenderingTypePredictor(RenderingTypePredictor):
def __init__(self) -> None:
super().__init__()
self._learning_data = list[tuple[Request, RenderingType]]()
def predict(self, request: Request) -> RenderingTypePrediction:
# Some custom logic that produces some `RenderingTypePrediction`
# based on the `request` input.
rendering_type: RenderingType = (
'static' if 'abc' in request.url else 'client only'
)
return RenderingTypePrediction(
# Recommends `static` rendering type -> HTTP-based sub crawler will be used.
rendering_type=rendering_type,
# Recommends that both sub crawlers should run with 20% chance. When both sub
# crawlers are running, the predictor can compare results and learn.
# High number means that predictor is not very confident about the
# `rendering_type`, low number means that predictor is very confident.
detection_probability_recommendation=0.2,
)
def store_result(self, request: Request, rendering_type: RenderingType) -> None:
# This function allows predictor to store new learning data and retrain itself
# if needed. `request` is input for prediction and `rendering_type` is the correct
# prediction.
self._learning_data.append((request, rendering_type))
# retrain
def result_checker(result: RequestHandlerRunResult) -> bool:
# Some function that inspects produced `result` and returns `True` if the result
# is correct.
return bool(result) # Check something on result
def result_comparator(
result_1: RequestHandlerRunResult, result_2: RequestHandlerRunResult
) -> bool:
# Some function that inspects two results and returns `True` if they are
# considered equivalent. It is used when comparing results produced by HTTP-based
# sub crawler and playwright based sub crawler.
return (
result_1.push_data_calls == result_2.push_data_calls
) # For example compare `push_data` calls.
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
rendering_type_predictor=CustomRenderingTypePredictor(),
result_checker=result_checker,
result_comparator=result_comparator,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,39 @@
import asyncio
from playwright.async_api import Route
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
AdaptivePlaywrightPreNavCrawlingContext,
)
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser()
@crawler.pre_navigation_hook
async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
"""Hook executed both in static sub crawler and playwright sub crawler.
Trying to access `context.page` in this hook would raise `AdaptiveContextError`
for pages crawled without playwright.
"""
context.log.info(f'pre navigation hook for: {context.request.url}')
@crawler.pre_navigation_hook(playwright_only=True)
async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
"""Hook executed only in playwright sub crawler."""
async def some_routing_function(route: Route) -> None:
await route.continue_()
await context.page.route('*/**', some_routing_function)
context.log.info(
f'Playwright only pre navigation hook for: {context.request.url}'
)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,28 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,28 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,32 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.soup.title.string if context.soup.title else None,
}
context.log.info(f'Extracted data: {data}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,32 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Extract data from the page.
data = {
'url': context.request.url,
'title': await context.page.title(),
}
context.log.info(f'Extracted data: {data}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
# The proxy URLs are rotated in a round-robin.
proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/
proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/
proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,24 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
use_session_pool=True,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,24 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
use_session_pool=True,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,39 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
tiered_proxy_urls=[
# No proxy tier.
# Optional in case you do not want to use any proxy on lowest tier.
[None],
# lower tier, cheaper, preferred as long as they work
[
'http://cheap-datacenter-proxy-1.com/',
'http://cheap-datacenter-proxy-2.com/',
],
# higher tier, more expensive, used as a fallback
[
'http://expensive-residential-proxy-1.com/',
'http://expensive-residential-proxy-2.com/',
],
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,39 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
tiered_proxy_urls=[
# No proxy tier.
# Optional in case you do not want to use any proxy on lowest tier.
[None],
# lower tier, cheaper, preferred as long as they work
[
'http://cheap-datacenter-proxy-1.com/',
'http://cheap-datacenter-proxy-2.com/',
],
# higher tier, more expensive, used as a fallback
[
'http://expensive-residential-proxy-1.com/',
'http://expensive-residential-proxy-2.com/',
],
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,28 @@
import asyncio
from crawlee.request_loaders import RequestList
async def main() -> None:
# Open the request list, if it does not exist, it will be created.
# Leave name empty to use the default request list.
request_list = RequestList(
name='my-request-list',
requests=[
'https://apify.com/',
'https://crawlee.dev/',
'https://crawlee.dev/python/',
],
)
# Fetch and process requests from the queue.
while request := await request_list.fetch_next_request():
# Do something with it...
print(f'Processing {request.url}')
# And mark it as handled.
await request_list.mark_request_as_handled(request)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,46 @@
import asyncio
import logging
from crawlee import service_locator
from crawlee.request_loaders import RequestList
logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
logger = logging.getLogger(__name__)
# Disable clearing the `KeyValueStore` on each run.
# This is necessary so that the state keys are not cleared at startup.
# The recommended way to achieve this behavior is setting the environment variable
# `CRAWLEE_PURGE_ON_START=0`
configuration = service_locator.get_configuration()
configuration.purge_on_start = False
async def main() -> None:
# Open the request list, if it does not exist, it will be created.
# Leave name empty to use the default request list.
request_list = RequestList(
name='my-request-list',
requests=[
'https://apify.com/',
'https://crawlee.dev/',
'https://crawlee.dev/python/',
],
# Enable persistence
persist_state_key='my-persist-state',
persist_requests_key='my-persist-requests',
)
# We receive only one request.
# Each time you run it, it will be a new request until you exhaust the `RequestList`.
request = await request_list.fetch_next_request()
if request:
logger.info(f'Processing request: {request.url}')
# Do something with it...
# And mark it as handled.
await request_list.mark_request_as_handled(request)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,43 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.request_loaders import RequestList
async def main() -> None:
# Create a static request list.
request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
# highlight-start
# Convert the request list to a request manager using the to_tandem method.
# It is a tandem with the default request queue.
request_manager = await request_list.to_tandem()
# highlight-end
# Create a crawler and pass the request manager to it.
crawler = ParselCrawler(
request_manager=request_manager,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.router.default_handler
async def handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# New links will be enqueued directly to the queue.
await context.enqueue_links()
# Extract data using Parsel's XPath and CSS selectors.
data = {
'url': context.request.url,
'title': context.selector.xpath('//title/text()').get(),
}
# Push extracted data to the dataset.
await context.push_data(data)
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,44 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.request_loaders import RequestList, RequestManagerTandem
from crawlee.storages import RequestQueue
async def main() -> None:
# Create a static request list.
request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
# Open the default request queue.
request_queue = await RequestQueue.open()
# And combine them together to a single request manager.
request_manager = RequestManagerTandem(request_list, request_queue)
# Create a crawler and pass the request manager to it.
crawler = ParselCrawler(
request_manager=request_manager,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.router.default_handler
async def handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# New links will be enqueued directly to the queue.
await context.enqueue_links()
# Extract data using Parsel's XPath and CSS selectors.
data = {
'url': context.request.url,
'title': context.selector.xpath('//title/text()').get(),
}
# Push extracted data to the dataset.
await context.push_data(data)
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,30 @@
import asyncio
import re
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import SitemapRequestLoader
async def main() -> None:
# Create an HTTP client for fetching the sitemap.
http_client = ImpitHttpClient()
# Create a sitemap request loader with filtering rules.
sitemap_loader = SitemapRequestLoader(
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
http_client=http_client,
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
)
# We work with the loader until we process all relevant links from the sitemap.
while request := await sitemap_loader.fetch_next_request():
# Do something with it...
print(f'Processing {request.url}')
# And mark it as handled.
await sitemap_loader.mark_request_as_handled(request)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,45 @@
import asyncio
import logging
from crawlee import service_locator
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import SitemapRequestLoader
logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
logger = logging.getLogger(__name__)
# Disable clearing the `KeyValueStore` on each run.
# This is necessary so that the state keys are not cleared at startup.
# The recommended way to achieve this behavior is setting the environment variable
# `CRAWLEE_PURGE_ON_START=0`
configuration = service_locator.get_configuration()
configuration.purge_on_start = False
async def main() -> None:
# Create an HTTP client for fetching sitemaps
# Use the context manager for `SitemapRequestLoader` to correctly save the state when
# the work is completed.
async with (
ImpitHttpClient() as http_client,
SitemapRequestLoader(
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
http_client=http_client,
# Enable persistence
persist_state_key='my-persist-state',
) as sitemap_loader,
):
# We receive only one request.
# Each time you run it, it will be a new request until you exhaust the sitemap.
request = await sitemap_loader.fetch_next_request()
if request:
logger.info(f'Processing request: {request.url}')
# Do something with it...
# And mark it as handled.
await sitemap_loader.mark_request_as_handled(request)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,53 @@
import asyncio
import re
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import SitemapRequestLoader
async def main() -> None:
# Create an HTTP client for fetching the sitemap.
http_client = ImpitHttpClient()
# Create a sitemap request loader with filtering rules.
sitemap_loader = SitemapRequestLoader(
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
http_client=http_client,
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
)
# highlight-start
# Convert the sitemap loader into a request manager linked
# to the default request queue.
request_manager = await sitemap_loader.to_tandem()
# highlight-end
# Create a crawler and pass the request manager to it.
crawler = ParselCrawler(
request_manager=request_manager,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.router.default_handler
async def handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# New links will be enqueued directly to the queue.
await context.enqueue_links()
# Extract data using Parsel's XPath and CSS selectors.
data = {
'url': context.request.url,
'title': context.selector.xpath('//title/text()').get(),
}
# Push extracted data to the dataset.
await context.push_data(data)
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,54 @@
import asyncio
import re
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import ImpitHttpClient
from crawlee.request_loaders import RequestManagerTandem, SitemapRequestLoader
from crawlee.storages import RequestQueue
async def main() -> None:
# Create an HTTP client for fetching the sitemap.
http_client = ImpitHttpClient()
# Create a sitemap request loader with filtering rules.
sitemap_loader = SitemapRequestLoader(
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
http_client=http_client,
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
)
# Open the default request queue.
request_queue = await RequestQueue.open()
# And combine them together to a single request manager.
request_manager = RequestManagerTandem(sitemap_loader, request_queue)
# Create a crawler and pass the request manager to it.
crawler = ParselCrawler(
request_manager=request_manager,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.router.default_handler
async def handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# New links will be enqueued directly to the queue.
await context.enqueue_links()
# Extract data using Parsel's XPath and CSS selectors.
data = {
'url': context.request.url,
'title': context.selector.xpath('//title/text()').get(),
}
# Push extracted data to the dataset.
await context.push_data(data)
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,51 @@
import asyncio
from crawlee import HttpHeaders
from crawlee.crawlers import (
AdaptivePlaywrightCrawler,
AdaptivePlaywrightCrawlingContext,
AdaptivePlaywrightPreNavCrawlingContext,
)
async def main() -> None:
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.pre_navigation_hook
async def common_setup(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
# Common pre-navigation hook - runs for both HTTP and browser requests.
context.request.headers |= HttpHeaders(
{'Accept': 'text/html,application/xhtml+xml'},
)
@crawler.pre_navigation_hook(playwright_only=True)
async def browser_setup(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
# Playwright-specific pre-navigation hook - runs only when browser is used.
await context.page.set_viewport_size({'width': 1280, 'height': 720})
if context.block_requests:
await context.block_requests(extra_url_patterns=['*.css', '*.js'])
@crawler.router.default_handler
async def default_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
# Extract title using the unified context interface.
title_tag = context.parsed_content.find('title')
title = title_tag.get_text() if title_tag else None
# Extract other data consistently across both modes.
links = [a.get('href') for a in context.parsed_content.find_all('a', href=True)]
await context.push_data(
{
'url': context.request.url,
'title': title,
'links': links,
}
)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,92 @@
import asyncio
from crawlee import Request
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.router import Router
async def main() -> None:
# Create a custom router instance
router = Router[ParselCrawlingContext]()
# Define the default handler (fallback for requests without specific labels)
@router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing homepage: {context.request.url}')
# Extract page title
title = context.selector.css('title::text').get() or 'No title found'
await context.push_data(
{
'url': context.request.url,
'title': title,
'page_type': 'homepage',
}
)
# Find and enqueue collection/category links
await context.enqueue_links(selector='a[href*="/collections/"]', label='CATEGORY')
# Define a handler for category pages
@router.handler('CATEGORY')
async def category_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing category page: {context.request.url}')
# Extract category information
category_title = context.selector.css('h1::text').get() or 'Unknown Category'
product_count = len(context.selector.css('.product-item').getall())
await context.push_data(
{
'url': context.request.url,
'type': 'category',
'category_title': category_title,
'product_count': product_count,
'handler': 'category',
}
)
# Enqueue product links from this category
await context.enqueue_links(selector='a[href*="/products/"]', label='PRODUCT')
# Define a handler for product detail pages
@router.handler('PRODUCT')
async def product_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing product page: {context.request.url}')
# Extract detailed product information
product_data = {
'url': context.request.url,
'name': context.selector.css('h1::text').get(),
'price': context.selector.css('.price::text').get(),
'description': context.selector.css('.product-description p::text').get(),
'images': context.selector.css('.product-gallery img::attr(src)').getall(),
'in_stock': bool(context.selector.css('.add-to-cart-button').get()),
'handler': 'product',
}
await context.push_data(product_data)
# Create crawler with the router
crawler = ParselCrawler(
request_handler=router,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
# Start crawling with some initial requests
await crawler.run(
[
# Will use default handler
'https://warehouse-theme-metal.myshopify.com/',
# Will use category handler
Request.from_url(
'https://warehouse-theme-metal.myshopify.com/collections/all',
label='CATEGORY',
),
]
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,44 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.router import Router
async def main() -> None:
# Create a custom router instance
router = Router[ParselCrawlingContext]()
# Define only a default handler
@router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract page title
title = context.selector.css('title::text').get() or 'No title found'
# Extract and save basic page data
await context.push_data(
{
'url': context.request.url,
'title': title,
}
)
# Find and enqueue product links for further crawling
await context.enqueue_links(
selector='a[href*="/products/"]',
label='PRODUCT', # Note: no handler for this label, will use default
)
# Create crawler with the custom router
crawler = ParselCrawler(
request_handler=router,
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
# Start crawling
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,62 @@
import asyncio
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
from crawlee.errors import HttpStatusCodeError
# HTTP status code constants
TOO_MANY_REQUESTS = 429
async def main() -> None:
# Create a crawler instance
crawler = ParselCrawler(
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract product information (might fail for some pages)
product_name = context.selector.css('h1[data-testid="product-title"]::text').get()
if not product_name:
raise ValueError('Product name not found - might be a non-product page')
price = context.selector.css('.price::text').get()
await context.push_data(
{
'url': context.request.url,
'product_name': product_name,
'price': price,
}
)
# Error handler - called when an error occurs during request processing
@crawler.error_handler
async def error_handler(context: BasicCrawlingContext, error: Exception) -> None:
error_name = type(error).__name__
context.log.warning(f'Error occurred for {context.request.url}: {error_name}')
# You can modify the request or context here before retry
if (
isinstance(error, HttpStatusCodeError)
and error.status_code == TOO_MANY_REQUESTS
):
context.log.info('Rate limited - will retry with delay')
# You could modify headers, add delay, etc.
elif isinstance(error, ValueError):
context.log.info('Parse error - marking request as no retry')
context.request.no_retry = True
# Start crawling
await crawler.run(
[
'https://warehouse-theme-metal.myshopify.com/products/on-running-cloudmonster-2-mens',
# Might cause parse error
'https://warehouse-theme-metal.myshopify.com/collections/mens-running',
]
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,64 @@
import asyncio
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
async def main() -> None:
# Create a crawler instance with retry settings
crawler = ParselCrawler(
max_requests_per_crawl=10, # Limit the max requests per crawl.
max_request_retries=2, # Allow 2 retries before failing
)
@crawler.router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract product information
product_name = context.selector.css('h1[data-testid="product-title"]::text').get()
if not product_name:
product_name = context.selector.css('h1::text').get() or 'Unknown Product'
price = context.selector.css('.price::text').get() or 'Price not available'
await context.push_data(
{
'url': context.request.url,
'product_name': product_name,
'price': price,
'status': 'success',
}
)
# Failed request handler - called when request has exhausted all retries
@crawler.failed_request_handler
async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
context.log.error(
f'Failed to process {context.request.url} after all retries: {error}'
)
# Save failed request information for analysis
await context.push_data(
{
'failed_url': context.request.url,
'label': context.request.label,
'error_type': type(error).__name__,
'error_message': str(error),
'retry_count': context.request.retry_count,
'status': 'failed',
}
)
# Start crawling with some URLs that might fail
await crawler.run(
[
'https://warehouse-theme-metal.myshopify.com/products/on-running-cloudmonster-2-mens',
# This will likely fail
'https://warehouse-theme-metal.myshopify.com/invalid-url',
'https://warehouse-theme-metal.myshopify.com/products/valid-product',
]
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,37 @@
import asyncio
from crawlee import HttpHeaders
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
async def main() -> None:
crawler = ParselCrawler(
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.pre_navigation_hook
async def setup_request(context: BasicCrawlingContext) -> None:
# Add custom headers before making the request
context.request.headers |= HttpHeaders(
{
'User-Agent': 'Crawlee Bot 1.0',
'Accept': 'text/html,application/xhtml+xml',
},
)
@crawler.router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
# Extract basic page information
title = context.selector.css('title::text').get()
await context.push_data(
{
'url': context.request.url,
'title': title,
}
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,59 @@
import asyncio
from crawlee.crawlers import (
PlaywrightCrawler,
PlaywrightCrawlingContext,
PlaywrightPreNavCrawlingContext,
)
async def main() -> None:
crawler = PlaywrightCrawler(
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
@crawler.pre_navigation_hook
async def setup_page(context: PlaywrightPreNavCrawlingContext) -> None:
# Set viewport size for consistent rendering
await context.page.set_viewport_size({'width': 1280, 'height': 720})
# Block unnecessary resources to speed up crawling
await context.block_requests(
extra_url_patterns=[
'*.png',
'*.jpg',
'*.jpeg',
'*.gif',
'*.svg',
'*.css',
'*.woff',
'*.woff2',
'*.ttf',
'*google-analytics*',
'*facebook*',
'*twitter*',
]
)
# Set custom user agent
await context.page.set_extra_http_headers(
{
'User-Agent': 'Mozilla/5.0 (compatible; Crawlee Bot)',
}
)
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
title = await context.page.title()
await context.push_data(
{
'url': context.request.url,
'title': title,
}
)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,50 @@
import asyncio
import time
from crawlee import Request
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.router import Router
async def main() -> None:
# Create a custom router instance
router = Router[ParselCrawlingContext]()
# Register a middleware that logs every request before it reaches a handler
@router.use
async def logging_middleware(context: ParselCrawlingContext) -> None:
context.log.info(
f'Processing request: {context.request.url} label={context.request.label}'
)
# Register a middleware that adds a timestamp to the request's user data
@router.use
async def timestamp_middleware(context: ParselCrawlingContext) -> None:
context.request.user_data['start_time'] = time.monotonic()
@router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} with default handler')
@router.handler('CATEGORY')
async def category_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} with category handler')
crawler = ParselCrawler(
request_handler=router,
max_requests_per_crawl=10,
)
await crawler.run(
[
'https://warehouse-theme-metal.myshopify.com/',
Request.from_url(
'https://warehouse-theme-metal.myshopify.com/collections/all',
label='CATEGORY',
),
]
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,36 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
async def main() -> None:
# Create a crawler instance
crawler = ParselCrawler(
max_requests_per_crawl=10, # Limit the max requests per crawl.
)
# Use the crawler's built-in router to define a default handler
@crawler.router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Extract page title
title = context.selector.css('title::text').get() or 'No title found'
# Extract and save basic page data
await context.push_data(
{
'url': context.request.url,
'title': title,
}
)
# Find and enqueue product links for further crawling
await context.enqueue_links(selector='a[href*="/products/"]', label='PRODUCT')
# Start crawling
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import BasicCrawler, BasicCrawlingContext
from crawlee.request_loaders import ThrottlingRequestManager
from crawlee.storages import RequestQueue
async def main() -> None:
# Open the default request queue.
queue = await RequestQueue.open()
# Wrap it with ThrottlingRequestManager for specific domains. The throttler uses the
# same storage backend as the underlying queue.
throttler = ThrottlingRequestManager(
queue,
domains=['api.example.com', 'slow-site.org'],
request_manager_opener=RequestQueue.open,
)
# Pass the throttler as the crawler's request manager.
crawler = BasicCrawler(request_manager=throttler)
@crawler.router.default_handler
async def handler(context: BasicCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Add requests. Listed domains are routed directly to their throttled sub-managers.
# Others go to the inner manager.
await throttler.add_requests(
[
'https://api.example.com/data',
'https://api.example.com/users',
'https://slow-site.org/page1',
'https://fast-site.com/page1', # Not throttled
]
)
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,54 @@
import asyncio
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
from typing import TypedDict
from fastapi import FastAPI
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
class State(TypedDict):
"""State available in the app."""
crawler: ParselCrawler
requests_to_results: dict[str, asyncio.Future[dict[str, str]]]
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[State]:
# Start up code that runs once when the app starts
# Results will be stored in this dictionary
requests_to_results = dict[str, asyncio.Future[dict[str, str]]]()
crawler = ParselCrawler(
# Keep the crawler alive even when there are no more requests to process now.
# This makes the crawler wait for more requests to be added later.
keep_alive=True
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
title = context.selector.xpath('//title/text()').get() or ''
# Extract data from the page and save it to the result dictionary.
requests_to_results[context.request.unique_key].set_result(
{
'title': title,
}
)
# Start the crawler without awaiting it to finish
crawler.log.info(f'Starting crawler for the {app.title}')
run_task = asyncio.create_task(crawler.run([]))
# Make the crawler and the result dictionary available in the app state
yield {'crawler': crawler, 'requests_to_results': requests_to_results}
# Cleanup code that runs once when the app shuts down
crawler.stop()
# Wait for the crawler to finish
await run_task
@@ -0,0 +1,58 @@
from __future__ import annotations
import asyncio
from uuid import uuid4
from fastapi import FastAPI
from starlette.requests import Request
from starlette.responses import HTMLResponse
import crawlee
from .crawler import lifespan
app = FastAPI(lifespan=lifespan, title='Crawler app')
@app.get('/', response_class=HTMLResponse)
def index() -> str:
return """
<!DOCTYPE html>
<html>
<body>
<h1>Scraper server</h1>
<p>To scrape some page, visit "scrape" endpoint with url parameter.
For example:
<a href="/scrape?url=https://www.example.com">
/scrape?url=https://www.example.com
</a>
</p>
</body>
</html>
"""
@app.get('/scrape')
async def scrape_url(request: Request, url: str | None = None) -> dict:
if not url:
return {'url': 'missing', 'scrape result': 'no results'}
# Generate random unique key for the request
unique_key = str(uuid4())
# Set the result future in the result dictionary so that it can be awaited
request.state.requests_to_results[unique_key] = asyncio.Future[dict[str, str]]()
# Add the request to the crawler queue
await request.state.crawler.add_requests(
[crawlee.Request.from_url(url, unique_key=unique_key)]
)
# Wait for the result future to be finished
result = await request.state.requests_to_results[unique_key]
# Clean the result from the result dictionary to free up memory
request.state.requests_to_results.pop(unique_key)
# Return the result
return {'url': url, 'scrape result': result}
@@ -0,0 +1,25 @@
import asyncio
from crawlee import ConcurrencySettings
from crawlee.crawlers import BeautifulSoupCrawler
async def main() -> None:
concurrency_settings = ConcurrencySettings(
# Set the maximum number of concurrent requests the crawler can run to 100.
max_concurrency=100,
# Limit the total number of requests to 10 per minute to avoid overwhelming
# the target website.
max_tasks_per_minute=10,
)
crawler = BeautifulSoupCrawler(
# Apply the defined concurrency settings to the crawler.
concurrency_settings=concurrency_settings,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,27 @@
import asyncio
from crawlee import ConcurrencySettings
from crawlee.crawlers import BeautifulSoupCrawler
async def main() -> None:
concurrency_settings = ConcurrencySettings(
# Start with 8 concurrent tasks, as long as resources are available.
desired_concurrency=8,
# Maintain a minimum of 5 concurrent tasks to ensure steady crawling.
min_concurrency=5,
# Limit the maximum number of concurrent tasks to 10 to prevent
# overloading the system.
max_concurrency=10,
)
crawler = BeautifulSoupCrawler(
# Use the configured concurrency settings for the crawler.
concurrency_settings=concurrency_settings,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,22 @@
import asyncio
from crawlee import service_locator
from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient
async def main() -> None:
# Register the storage client via service locator.
memory_storage_client = MemoryStorageClient()
service_locator.set_storage_client(memory_storage_client)
# Retrieve the storage client.
current_storage_client = service_locator.get_storage_client()
# Try to set a different storage client, which will raise ServiceConflictError
# if storage client was already retrieved.
file_system_storage_client = FileSystemStorageClient()
service_locator.set_storage_client(file_system_storage_client)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,22 @@
import asyncio
from datetime import timedelta
from crawlee.configuration import Configuration
from crawlee.crawlers import ParselCrawler
async def main() -> None:
configuration = Configuration(
log_level='DEBUG',
headless=False,
persist_state_interval=timedelta(seconds=30),
)
# Register configuration via crawler.
crawler = ParselCrawler(
configuration=configuration,
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,20 @@
import asyncio
from datetime import timedelta
from crawlee.crawlers import ParselCrawler
from crawlee.events import LocalEventManager
async def main() -> None:
event_manager = LocalEventManager(
system_info_interval=timedelta(seconds=5),
)
# Register event manager via crawler.
crawler = ParselCrawler(
event_manager=event_manager,
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,17 @@
import asyncio
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import MemoryStorageClient
async def main() -> None:
storage_client = MemoryStorageClient()
# Register storage client via crawler.
crawler = ParselCrawler(
storage_client=storage_client,
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,20 @@
import asyncio
from datetime import timedelta
from crawlee import service_locator
from crawlee.configuration import Configuration
async def main() -> None:
configuration = Configuration(
log_level='DEBUG',
headless=False,
persist_state_interval=timedelta(seconds=30),
)
# Register configuration via service locator.
service_locator.set_configuration(configuration)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,18 @@
import asyncio
from datetime import timedelta
from crawlee import service_locator
from crawlee.events import LocalEventManager
async def main() -> None:
event_manager = LocalEventManager(
system_info_interval=timedelta(seconds=5),
)
# Register event manager via service locator.
service_locator.set_event_manager(event_manager)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,15 @@
import asyncio
from crawlee import service_locator
from crawlee.storage_clients import MemoryStorageClient
async def main() -> None:
storage_client = MemoryStorageClient()
# Register storage client via service locator.
service_locator.set_storage_client(storage_client)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,30 @@
import asyncio
from datetime import timedelta
from crawlee import service_locator
from crawlee.configuration import Configuration
from crawlee.storage_clients import MemoryStorageClient
from crawlee.storages import Dataset
async def main() -> None:
configuration = Configuration(
log_level='DEBUG',
headless=False,
persist_state_interval=timedelta(seconds=30),
)
# Set the custom configuration as the global default configuration.
service_locator.set_configuration(configuration)
# Use the global defaults when creating the dataset (or other storage).
dataset_1 = await Dataset.open()
# Or set explicitly specific configuration if
# you do not want to rely on global defaults.
dataset_2 = await Dataset.open(
storage_client=MemoryStorageClient(), configuration=configuration
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,17 @@
import asyncio
from crawlee.storage_clients import MemoryStorageClient
from crawlee.storages import Dataset
async def main() -> None:
storage_client = MemoryStorageClient()
# Pass the storage client to the dataset (or other storage) when opening it.
dataset = await Dataset.open(
storage_client=storage_client,
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,85 @@
import asyncio
from collections.abc import Callable
from datetime import timedelta
from itertools import count
from crawlee import ConcurrencySettings, Request
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import RequestCollisionError
from crawlee.sessions import Session, SessionPool
# Define a function for creating sessions with simple logic for unique `id` generation.
# This is necessary if you need to specify a particular session for the first request,
# for example during authentication
def create_session_function() -> Callable[[], Session]:
counter = count()
def create_session() -> Session:
return Session(
id=str(next(counter)),
max_usage_count=999_999,
max_age=timedelta(hours=999_999),
max_error_score=100,
blocked_status_codes=[403],
)
return create_session
async def main() -> None:
crawler = HttpCrawler(
# Adjust request limits according to your pool size
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=500),
# Requests are bound to specific sessions, no rotation needed
max_session_rotations=0,
session_pool=SessionPool(
max_pool_size=10, create_session_function=create_session_function()
),
)
@crawler.router.default_handler
async def basic_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Initialize the session and bind the next request to this session if needed
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
next_requests = []
if context.session:
context.log.info(f'Init session {context.session.id}')
next_request = Request.from_url(
'https://a.placeholder.com', session_id=context.session.id
)
next_requests.append(next_request)
await context.add_requests(next_requests)
# Handle errors when a session is blocked and no longer available in the pool
# when attempting to execute requests bound to it
@crawler.failed_request_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
if isinstance(error, RequestCollisionError) and context.session:
context.log.error(
f'Request {context.request.url} failed, because the bound '
'session is unavailable'
)
# Create a pool of requests bound to their respective sessions
# Use `always_enqueue=True` if session initialization happens on a non-unique address,
# such as the site's main page
init_requests = [
Request.from_url(
'https://example.org/',
label='session_init',
session_id=str(session_id),
use_extended_unique_key=True,
)
for session_id in range(1, 11)
]
await crawler.run(init_requests)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,56 @@
import asyncio
from datetime import timedelta
from crawlee import ConcurrencySettings, Request
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
from crawlee.errors import SessionError
from crawlee.sessions import SessionPool
async def main() -> None:
crawler = HttpCrawler(
# Limit requests per minute to reduce the chance of being blocked
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=50),
# Disable session rotation
max_session_rotations=0,
session_pool=SessionPool(
# Only one session in the pool
max_pool_size=1,
create_session_settings={
# High value for session usage limit
'max_usage_count': 999_999,
# High value for session lifetime
'max_age': timedelta(hours=999_999),
# High score allows the session to encounter more errors
# before crawlee decides the session is blocked
# Make sure you know how to handle these errors
'max_error_score': 100,
# 403 status usually indicates you're already blocked
'blocked_status_codes': [403],
},
),
)
# Basic request handling logic
@crawler.router.default_handler
async def basic_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Handler for session initialization (authentication, initial cookies, etc.)
@crawler.router.handler(label='session_init')
async def session_init(context: HttpCrawlingContext) -> None:
if context.session:
context.log.info(f'Init session {context.session.id}')
# Monitor if our session gets blocked and explicitly stop the crawler
@crawler.error_handler
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
if isinstance(error, SessionError) and context.session:
context.log.info(f'Session {context.session.id} blocked')
crawler.stop()
await crawler.run([Request.from_url('https://example.org/', label='session_init')])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,48 @@
import asyncio
import re
from crawlee.crawlers import BasicCrawler, BasicCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
async def main() -> None:
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxy_configuration = ProxyConfiguration(
# options
)
# Initialize crawler with a custom SessionPool configuration
# to manage concurrent sessions and proxy rotation
crawler = BasicCrawler(
proxy_configuration=proxy_configuration,
# Activates the Session pool (default is true).
use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
# Define the default request handler that manages session states
@crawler.router.default_handler
async def default_handler(context: BasicCrawlingContext) -> None:
# Send request, BasicCrawler automatically selects a session from the pool
# and sets a proxy for it. You can check it with `context.session`
# and `context.proxy_info`.
response = await context.send_request(context.request.url)
page_content = (await response.read()).decode()
title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
if context.session and (title := title_match.group(1) if title_match else None):
if title == 'Blocked':
context.session.retire()
elif title == 'Not sure if blocked, might also be a connection error':
context.session.mark_bad()
else:
context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
async def main() -> None:
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxy_configuration = ProxyConfiguration(
# options
)
# Initialize crawler with a custom SessionPool configuration
# to manage concurrent sessions and proxy rotation
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
# Activates the Session pool (default is true).
use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
# Define the default request handler that manages session states
# based on the response content and potential blocking
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
title = context.soup.title.get_text() if context.soup.title else None
if context.session:
if title == 'Blocked':
context.session.retire()
elif title == 'Not sure if blocked, might also be a connection error':
context.session.mark_bad()
else:
context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,44 @@
import asyncio
import re
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
async def main() -> None:
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxy_configuration = ProxyConfiguration(
# options
)
# Initialize crawler with a custom SessionPool configuration
# to manage concurrent sessions and proxy rotation
crawler = HttpCrawler(
proxy_configuration=proxy_configuration,
# Activates the Session pool (default is true).
use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
# Define the default request handler that manages session states
# based on the response content and potential blocking
@crawler.router.default_handler
async def default_handler(context: HttpCrawlingContext) -> None:
page_content = (await context.http_response.read()).decode()
title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
if context.session and (title := title_match.group(1) if title_match else None):
if title == 'Blocked':
context.session.retire()
elif title == 'Not sure if blocked, might also be a connection error':
context.session.mark_bad()
else:
context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
async def main() -> None:
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxy_configuration = ProxyConfiguration(
# options
)
# Initialize crawler with a custom SessionPool configuration
# to manage concurrent sessions and proxy rotation
crawler = ParselCrawler(
proxy_configuration=proxy_configuration,
# Activates the Session pool (default is true).
use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
# Define the default request handler that manages session states
# based on the response content and potential blocking
@crawler.router.default_handler
async def default_handler(context: ParselCrawlingContext) -> None:
title = context.selector.css('title::text').get()
if context.session:
if title == 'Blocked':
context.session.retire()
elif title == 'Not sure if blocked, might also be a connection error':
context.session.mark_bad()
else:
context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
async def main() -> None:
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
proxy_configuration = ProxyConfiguration(
# options
)
# Initialize crawler with a custom SessionPool configuration
# to manage concurrent sessions and proxy rotation
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
# Activates the Session pool (default is true).
use_session_pool=True,
# Overrides default Session pool configuration.
session_pool=SessionPool(max_pool_size=100),
)
# Define the default request handler that manages session states
# based on the response content and potential blocking
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
title = await context.page.title()
if context.session:
if title == 'Blocked':
context.session.retire()
elif title == 'Not sure if blocked, might also be a connection error':
context.session.mark_bad()
else:
context.session.mark_good() # BasicCrawler handles this automatically.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,25 @@
import asyncio
from crawlee.sessions import SessionPool
async def main() -> None:
# Override the default Session pool configuration.
async with SessionPool(
max_pool_size=100,
create_session_settings={'max_usage_count': 10, 'blocked_status_codes': [403]},
) as session_pool:
session = await session_pool.get_session()
# Increase the error_score.
session.mark_bad()
# Throw away the session.
session.retire()
# Lower the error_score and mark the session good.
session.mark_good()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,47 @@
import asyncio
from typing import cast
from crawlee.browsers import StagehandOptions
from crawlee.crawlers import StagehandCrawler, StagehandCrawlingContext
async def main() -> None:
crawler = StagehandCrawler(
stagehand_options=StagehandOptions(
model_api_key='your-openai-api-key',
model='openai/gpt-5.4-nano',
),
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def handler(context: StagehandCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Dismiss overlays or interact with the page using natural language.
await context.page.act(input='Click the accept cookies button if present')
# Extract data from the page using AI.
extracted = await context.page.extract(
instruction='Get the page title and the main heading text',
schema={
'type': 'object',
'properties': {
'title': {'type': 'string'},
'heading': {'type': 'string'},
},
},
)
extract_result = extracted.data.result
if isinstance(extract_result, dict):
# Push extracted data to the dataset
# Use `cast()` to provide a more specific type hint for the extracted data.
await context.push_data(cast('dict[str, str | None]', extract_result))
await crawler.run(['https://example.com'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,37 @@
import asyncio
from typing import cast
from crawlee.browsers import StagehandOptions
from crawlee.crawlers import StagehandCrawler, StagehandCrawlingContext
async def main() -> None:
# Use Browserbase cloud browser instead of a local Chromium instance.
crawler = StagehandCrawler(
stagehand_options=StagehandOptions(
env='BROWSERBASE',
browserbase_api_key='your-browserbase-api-key',
project_id='your-project-id',
model_api_key='your-openai-api-key',
model='openai/gpt-5.4-nano',
),
max_requests_per_crawl=5,
)
@crawler.router.default_handler
async def handler(context: StagehandCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
extracted = await context.page.extract(
instruction='Get the main content of the page',
)
extract_result = extracted.data.result
await context.push_data(cast('dict[str, str | None]', extract_result))
await crawler.run(['https://example.com'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,65 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from crawlee.storage_clients import StorageClient
from crawlee.storage_clients._base import (
DatasetClient,
KeyValueStoreClient,
RequestQueueClient,
)
if TYPE_CHECKING:
from crawlee.configuration import Configuration
# Implement the storage type clients with your backend logic.
class CustomDatasetClient(DatasetClient):
# Implement methods like push_data, get_data, iterate_items, etc.
pass
class CustomKeyValueStoreClient(KeyValueStoreClient):
# Implement methods like get_value, set_value, delete, etc.
pass
class CustomRequestQueueClient(RequestQueueClient):
# Implement methods like add_request, fetch_next_request, etc.
pass
# Implement the storage client factory.
class CustomStorageClient(StorageClient):
async def create_dataset_client(
self,
*,
id: str | None = None,
name: str | None = None,
configuration: Configuration | None = None,
) -> CustomDatasetClient:
# Create and return your custom dataset client.
pass
async def create_kvs_client(
self,
*,
id: str | None = None,
name: str | None = None,
configuration: Configuration | None = None,
) -> CustomKeyValueStoreClient:
# Create and return your custom key-value store client.
pass
async def create_rq_client(
self,
*,
id: str | None = None,
name: str | None = None,
configuration: Configuration | None = None,
) -> CustomRequestQueueClient:
# Create and return your custom request queue client.
pass
@@ -0,0 +1,8 @@
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import FileSystemStorageClient
# Create a new instance of storage client.
storage_client = FileSystemStorageClient()
# And pass it to the crawler.
crawler = ParselCrawler(storage_client=storage_client)
@@ -0,0 +1,18 @@
from crawlee.configuration import Configuration
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import FileSystemStorageClient
# Create a new instance of storage client.
storage_client = FileSystemStorageClient()
# Create a configuration with custom settings.
configuration = Configuration(
storage_dir='./my_storage',
purge_on_start=False,
)
# And pass them to the crawler.
crawler = ParselCrawler(
storage_client=storage_client,
configuration=configuration,
)
@@ -0,0 +1,8 @@
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import MemoryStorageClient
# Create a new instance of storage client.
storage_client = MemoryStorageClient()
# And pass it to the crawler.
crawler = ParselCrawler(storage_client=storage_client)
@@ -0,0 +1,10 @@
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import RedisStorageClient
# Create a new instance of storage client using connection string.
# 'redis://localhost:6379' is the just placeholder, replace it with your actual
# connection string.
storage_client = RedisStorageClient(connection_string='redis://localhost:6379')
# And pass it to the crawler.
crawler = ParselCrawler(storage_client=storage_client)
@@ -0,0 +1,27 @@
from redis.asyncio import Redis
from crawlee.configuration import Configuration
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import RedisStorageClient
# Create a new instance of storage client using a Redis client with custom settings.
# Replace host and port with your actual Redis server configuration.
# Other Redis client settings can be adjusted as needed.
storage_client = RedisStorageClient(
redis=Redis(
host='localhost',
port=6379,
retry_on_timeout=True,
socket_keepalive=True,
socket_connect_timeout=10,
)
)
# Create a configuration with custom settings.
configuration = Configuration(purge_on_start=False)
# And pass them to the crawler.
crawler = ParselCrawler(
storage_client=storage_client,
configuration=configuration,
)
@@ -0,0 +1,29 @@
import asyncio
from crawlee import service_locator
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import MemoryStorageClient
from crawlee.storages import Dataset
async def main() -> None:
# Create custom storage client, MemoryStorageClient for example.
storage_client = MemoryStorageClient()
# Register it globally via the service locator.
service_locator.set_storage_client(storage_client)
# Or pass it directly to the crawler, it will be registered globally
# to the service locator under the hood.
crawler = ParselCrawler(storage_client=storage_client)
# Or just provide it when opening a storage (e.g. dataset), it will be used
# for this storage only, not globally.
dataset = await Dataset.open(
name='my-dataset',
storage_client=storage_client,
)
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,12 @@
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import SqlStorageClient
async def main() -> None:
# Create a new instance of storage client.
# This will create an SQLite database file crawlee.db or created tables in your
# database if you pass `connection_string` or `engine`
# Use the context manager to ensure that connections are properly cleaned up.
async with SqlStorageClient() as storage_client:
# And pass it to the crawler.
crawler = ParselCrawler(storage_client=storage_client)
@@ -0,0 +1,33 @@
from sqlalchemy.ext.asyncio import create_async_engine
from crawlee.configuration import Configuration
from crawlee.crawlers import ParselCrawler
from crawlee.storage_clients import SqlStorageClient
async def main() -> None:
# Create a new instance of storage client.
# On first run, also creates tables in your PostgreSQL database.
# Use the context manager to ensure that connections are properly cleaned up.
async with SqlStorageClient(
# Create an `engine` with the desired configuration
engine=create_async_engine(
'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres',
future=True,
pool_size=5,
max_overflow=10,
pool_recycle=3600,
pool_pre_ping=True,
echo=False,
)
) as storage_client:
# Create a configuration with custom settings.
configuration = Configuration(
purge_on_start=False,
)
# And pass them to the crawler.
crawler = ParselCrawler(
storage_client=storage_client,
configuration=configuration,
)

Some files were not shown because too many files have changed in this diff Show More