同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.fingerprint_suite import (
|
||||
DefaultFingerprintGenerator,
|
||||
HeaderGeneratorOptions,
|
||||
ScreenOptions,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
fingerprint_generator = DefaultFingerprintGenerator(
|
||||
header_options=HeaderGeneratorOptions(browsers=['chrome']),
|
||||
screen_options=ScreenOptions(min_width=400),
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,81 @@
|
||||
import asyncio
|
||||
|
||||
# CloakBrowser is an external package. Install it separately.
|
||||
from cloakbrowser.config import IGNORE_DEFAULT_ARGS, get_default_stealth_args
|
||||
from cloakbrowser.download import ensure_binary
|
||||
from typing_extensions import override
|
||||
|
||||
from crawlee.browsers import (
|
||||
BrowserPool,
|
||||
PlaywrightBrowserController,
|
||||
PlaywrightBrowserPlugin,
|
||||
)
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
class CloakBrowserPlugin(PlaywrightBrowserPlugin):
|
||||
"""Example browser plugin that uses CloakBrowser's patched Chromium,
|
||||
but otherwise keeps the functionality of PlaywrightBrowserPlugin.
|
||||
"""
|
||||
|
||||
@override
|
||||
async def new_browser(self) -> PlaywrightBrowserController:
|
||||
if not self._playwright:
|
||||
raise RuntimeError('Playwright browser plugin is not initialized.')
|
||||
|
||||
binary_path = ensure_binary()
|
||||
stealth_args = get_default_stealth_args()
|
||||
|
||||
# Merge CloakBrowser stealth args with any user-provided launch options.
|
||||
launch_options = dict(self._browser_launch_options)
|
||||
launch_options.pop('executable_path', None)
|
||||
launch_options.pop('chromium_sandbox', None)
|
||||
existing_args = list(launch_options.pop('args', []))
|
||||
launch_options['args'] = [*existing_args, *stealth_args]
|
||||
|
||||
return PlaywrightBrowserController(
|
||||
browser=await self._playwright.chromium.launch(
|
||||
executable_path=binary_path,
|
||||
ignore_default_args=IGNORE_DEFAULT_ARGS,
|
||||
**launch_options,
|
||||
),
|
||||
max_open_pages_per_browser=1,
|
||||
# CloakBrowser handles fingerprints at the binary level.
|
||||
header_generator=None,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
# Custom browser pool. Gives users full control over browsers used by the crawler.
|
||||
browser_pool=BrowserPool(plugins=[CloakBrowserPlugin()]),
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Extract some data from the page using Playwright's API.
|
||||
posts = await context.page.query_selector_all('.athing')
|
||||
for post in posts:
|
||||
# Get the HTML elements for the title and rank within each post.
|
||||
title_element = await post.query_selector('.title a')
|
||||
|
||||
# Extract the data we want from the elements.
|
||||
title = await title_element.inner_text() if title_element else None
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data({'title': title})
|
||||
|
||||
# Find a link to the next page and enqueue it if it exists.
|
||||
await context.enqueue_links(selector='.morelink')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://news.ycombinator.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,23 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Fingerprint generator is used by default.
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Find a link to the next page and enqueue it if it exists.
|
||||
await context.enqueue_links(selector='.morelink')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://news.ycombinator.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,61 @@
|
||||
import asyncio
|
||||
import io
|
||||
from pathlib import Path
|
||||
|
||||
from warcio.statusandheaders import StatusAndHeaders
|
||||
from warcio.warcwriter import WARCWriter
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
async def archive_response(context: ParselCrawlingContext, writer: WARCWriter) -> None:
|
||||
"""Helper function for archiving response in WARC format."""
|
||||
# Create WARC records for response
|
||||
response_body = await context.http_response.read()
|
||||
response_payload_stream = io.BytesIO(response_body)
|
||||
|
||||
response_headers = StatusAndHeaders(
|
||||
str(context.http_response.status_code),
|
||||
context.http_response.headers,
|
||||
protocol='HTTP/1.1',
|
||||
)
|
||||
response_record = writer.create_warc_record(
|
||||
context.request.url,
|
||||
'response',
|
||||
payload=response_payload_stream,
|
||||
length=len(response_body),
|
||||
http_headers=response_headers,
|
||||
)
|
||||
writer.write_record(response_record)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = ParselCrawler(
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Create a WARC archive file a prepare the writer.
|
||||
archive = Path('example.warc.gz')
|
||||
with archive.open('wb') as output:
|
||||
writer = WARCWriter(output, gzip=True)
|
||||
|
||||
# Create a WARC info record to store metadata about the archive.
|
||||
warcinfo_payload = {
|
||||
'software': 'Crawlee',
|
||||
'format': 'WARC/1.1',
|
||||
'description': 'Example archive created with ParselCrawler',
|
||||
}
|
||||
writer.write_record(writer.create_warcinfo_record(archive.name, warcinfo_payload))
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Archiving {context.request.url} ...')
|
||||
await archive_response(context=context, writer=writer)
|
||||
await context.enqueue_links(strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
|
||||
from playwright.async_api import Request
|
||||
from warcio.statusandheaders import StatusAndHeaders
|
||||
from warcio.warcwriter import WARCWriter
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def archive_response(
|
||||
request: Request, writer: WARCWriter, logger: logging.Logger
|
||||
) -> None:
|
||||
"""Helper function for archiving response in WARC format."""
|
||||
response = await request.response()
|
||||
if not response:
|
||||
logger.warning(f'Could not get response {request.url}')
|
||||
return
|
||||
try:
|
||||
response_body = await response.body()
|
||||
except Exception as e:
|
||||
logger.warning(f'Could not get response body for {response.url}: {e}')
|
||||
return
|
||||
logger.info(f'Archiving resource {response.url}')
|
||||
response_payload_stream = io.BytesIO(response_body)
|
||||
response_headers = StatusAndHeaders(
|
||||
str(response.status), response.headers, protocol='HTTP/1.1'
|
||||
)
|
||||
response_record = writer.create_warc_record(
|
||||
response.url,
|
||||
'response',
|
||||
payload=response_payload_stream,
|
||||
length=len(response_body),
|
||||
http_headers=response_headers,
|
||||
)
|
||||
writer.write_record(response_record)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
max_requests_per_crawl=1,
|
||||
headless=False,
|
||||
)
|
||||
|
||||
# Create a WARC archive file a prepare the writer.
|
||||
archive = Path('example.warc.gz')
|
||||
with archive.open('wb') as output:
|
||||
writer = WARCWriter(output, gzip=True)
|
||||
|
||||
# Create a WARC info record to store metadata about the archive.
|
||||
warcinfo_payload = {
|
||||
'software': 'Crawlee',
|
||||
'format': 'WARC/1.1',
|
||||
'description': 'Example archive created with PlaywrightCrawler',
|
||||
}
|
||||
writer.write_record(writer.create_warcinfo_record(archive.name, warcinfo_payload))
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def archiving_hook(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
# Ensure that all responses with additional resources are archived
|
||||
context.page.on(
|
||||
'requestfinished',
|
||||
partial(archive_response, logger=context.log, writer=writer),
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# For some sites, where the content loads dynamically,
|
||||
# it is needed to scroll the page to load all content.
|
||||
# It slows down the crawling, but ensures that all content is loaded.
|
||||
await context.infinite_scroll()
|
||||
await context.enqueue_links(strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Use the local wayback server as a proxy
|
||||
proxy_configuration=ProxyConfiguration(proxy_urls=['http://localhost:8080/']),
|
||||
# Ignore the HTTPS errors if you have not followed pywb CA setup instructions
|
||||
browser_launch_options={'ignore_https_errors': True},
|
||||
max_requests_per_crawl=10,
|
||||
headless=False,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Archiving {context.request.url} ...')
|
||||
# For some sites, where the content loads dynamically,
|
||||
# it is needed to scroll the page to load all content.
|
||||
# It slows down the crawling, but ensures that all content is loaded.
|
||||
await context.infinite_scroll()
|
||||
await context.enqueue_links(strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,45 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from crawlee import HttpHeaders
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import HttpStatusCodeError
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
# Using a placeholder refresh token for this example
|
||||
REFRESH_TOKEN = 'PLACEHOLDER'
|
||||
UNAUTHORIZED_CODE = 401
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=2,
|
||||
# Only treat 403 as a blocking status code, not 401
|
||||
session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}),
|
||||
# Don't treat 401 responses as errors
|
||||
ignore_http_error_status_codes=[UNAUTHORIZED_CODE],
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
# Now we can handle 401 responses ourselves
|
||||
if context.http_response.status_code == UNAUTHORIZED_CODE:
|
||||
# Get a fresh access token
|
||||
headers = {'authorization': f'Bearer {REFRESH_TOKEN}'}
|
||||
response = await context.send_request(
|
||||
'https://placeholder.org/refresh', headers=headers
|
||||
)
|
||||
data = json.loads(await response.read())
|
||||
# Add the new token to our `Request` headers
|
||||
context.request.headers |= HttpHeaders(
|
||||
{'authorization': f'Bearer {data["access_token"]}'},
|
||||
)
|
||||
# Trigger a retry with our updated headers
|
||||
raise HttpStatusCodeError('Unauthorized', status_code=UNAUTHORIZED_CODE)
|
||||
|
||||
await crawler.run(['http://httpbingo.org/status/401'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import HttpStatusCodeError, SessionError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(max_request_retries=5)
|
||||
|
||||
# Create a parsing error for demonstration
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
raise ValueError('Simulated parsing error')
|
||||
|
||||
# This handler runs before any retry attempts
|
||||
@crawler.error_handler
|
||||
async def retry_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
context.log.error(f'Failed request {context.request.url}')
|
||||
# Only allow retries for network-related errors
|
||||
if not isinstance(error, (SessionError, HttpStatusCodeError)):
|
||||
context.log.error('Non-network error detected')
|
||||
# Stop further retry attempts for this `Request`
|
||||
context.request.no_retry = True
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import ProxyError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Set how many session rotations will happen before calling the error handler
|
||||
# when ProxyError occurs
|
||||
crawler = HttpCrawler(max_session_rotations=5, max_request_retries=6)
|
||||
|
||||
# For this example, we'll create a proxy error in our handler
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
raise ProxyError('Simulated proxy error')
|
||||
|
||||
# This handler runs after all retry attempts are exhausted
|
||||
@crawler.failed_request_handler
|
||||
async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
context.log.error(f'Failed request {context.request.url}, after 5 rotations')
|
||||
request = context.request
|
||||
# For proxy errors, we can add a new `Request` to try again
|
||||
if isinstance(error, ProxyError) and not request.unique_key.startswith('retry'):
|
||||
context.log.info(f'Retrying {request.url} ...')
|
||||
# Create a new `Request` with a modified key to avoid deduplication
|
||||
new_request = Request.from_url(
|
||||
request.url, unique_key=f'retry{request.unique_key}'
|
||||
)
|
||||
|
||||
# Add the new `Request` to the `Queue`
|
||||
rq = await crawler.get_request_manager()
|
||||
await rq.add_request(new_request)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.http_clients import CurlImpersonateHttpClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
http_client = CurlImpersonateHttpClient(
|
||||
# Optional additional keyword arguments for `curl_cffi.requests.AsyncSession`.
|
||||
timeout=10,
|
||||
impersonate='chrome131',
|
||||
)
|
||||
|
||||
crawler = ParselCrawler(
|
||||
http_client=http_client,
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links from the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.css('title::text').get(),
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.http_clients import HttpxHttpClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
http_client = HttpxHttpClient(
|
||||
# Optional additional keyword arguments for `httpx.AsyncClient`.
|
||||
timeout=10,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
crawler = ParselCrawler(
|
||||
http_client=http_client,
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links from the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.css('title::text').get(),
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
http_client = ImpitHttpClient(
|
||||
# Optional additional keyword arguments for `impit.AsyncClient`.
|
||||
http3=True,
|
||||
browser='firefox',
|
||||
verify=True,
|
||||
)
|
||||
|
||||
crawler = ParselCrawler(
|
||||
http_client=http_client,
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Enqueue all links from the page.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.css('title::text').get(),
|
||||
}
|
||||
|
||||
# Push the extracted data to the default dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a BeautifulSoupCrawler instance
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Limit the crawl to 10 requests
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract data using BeautifulSoup
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset
|
||||
await context.push_data(data)
|
||||
|
||||
# Enqueue links found on the page for further crawling
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,52 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an HttpCrawler instance - no automatic parsing
|
||||
crawler = HttpCrawler(
|
||||
# Limit the crawl to 10 requests
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Get the raw response content
|
||||
response_body = await context.http_response.read()
|
||||
response_text = response_body.decode('utf-8')
|
||||
|
||||
# Extract title manually using regex (since we don't have a parser)
|
||||
title_match = re.search(
|
||||
r'<title[^>]*>([^<]+)</title>', response_text, re.IGNORECASE
|
||||
)
|
||||
title = title_match.group(1).strip() if title_match else None
|
||||
|
||||
# Extract basic information
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset
|
||||
await context.push_data(data)
|
||||
|
||||
# Simple link extraction for further crawling
|
||||
href_pattern = r'href=["\']([^"\']+)["\']'
|
||||
matches = re.findall(href_pattern, response_text, re.IGNORECASE)
|
||||
|
||||
# Enqueue first few links found (limit to avoid too many requests)
|
||||
for href in matches[:3]:
|
||||
if href.startswith('http') and 'crawlee.dev' in href:
|
||||
await context.add_requests([href])
|
||||
|
||||
# Run the crawler
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,63 @@
|
||||
import asyncio
|
||||
|
||||
from pydantic import ValidationError
|
||||
from selectolax.lexbor import LexborHTMLParser
|
||||
from yarl import URL
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=1,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Parse the HTML content using Selectolax with Lexbor backend.
|
||||
parsed_html = LexborHTMLParser(await context.http_response.read())
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': parsed_html.css_first('title').text(),
|
||||
'h1s': [h1.text() for h1 in parsed_html.css('h1')],
|
||||
'h2s': [h2.text() for h2 in parsed_html.css('h2')],
|
||||
'h3s': [h3.text() for h3 in parsed_html.css('h3')],
|
||||
}
|
||||
await context.push_data(data)
|
||||
|
||||
# Css selector to extract valid href attributes.
|
||||
links_selector = (
|
||||
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
|
||||
)
|
||||
base_url = URL(context.request.url)
|
||||
extracted_requests = []
|
||||
|
||||
# Extract links.
|
||||
for item in parsed_html.css(links_selector):
|
||||
href = item.attributes.get('href')
|
||||
if not href:
|
||||
continue
|
||||
|
||||
# Convert relative URLs to absolute if needed.
|
||||
url = str(base_url.join(URL(href)))
|
||||
try:
|
||||
request = Request.from_url(url)
|
||||
except ValidationError as exc:
|
||||
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
|
||||
continue
|
||||
extracted_requests.append(request)
|
||||
|
||||
# Add extracted requests to the queue with the same-domain strategy.
|
||||
await context.add_requests(extracted_requests, strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,61 @@
|
||||
import asyncio
|
||||
|
||||
from lxml import html
|
||||
from pydantic import ValidationError
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=1,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Parse the HTML content using lxml.
|
||||
parsed_html = html.fromstring(await context.http_response.read())
|
||||
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': parsed_html.findtext('.//title'),
|
||||
'h1s': [h1.text_content() for h1 in parsed_html.findall('.//h1')],
|
||||
'h2s': [h2.text_content() for h2 in parsed_html.findall('.//h2')],
|
||||
'h3s': [h3.text_content() for h3 in parsed_html.findall('.//h3')],
|
||||
}
|
||||
await context.push_data(data)
|
||||
|
||||
# Convert relative URLs to absolute before extracting links.
|
||||
parsed_html.make_links_absolute(context.request.url, resolve_base_href=True)
|
||||
|
||||
# Xpath 1.0 selector for extracting valid href attributes.
|
||||
links_xpath = (
|
||||
'//a/@href[not(starts-with(., "#")) '
|
||||
'and not(starts-with(., "javascript:")) '
|
||||
'and not(starts-with(., "mailto:"))]'
|
||||
)
|
||||
|
||||
extracted_requests = []
|
||||
|
||||
# Extract links.
|
||||
for url in parsed_html.xpath(links_xpath):
|
||||
try:
|
||||
request = Request.from_url(url)
|
||||
except ValidationError as exc:
|
||||
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
|
||||
continue
|
||||
extracted_requests.append(request)
|
||||
|
||||
# Add extracted requests to the queue with the same-domain strategy.
|
||||
await context.add_requests(extracted_requests, strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
import asyncio
|
||||
|
||||
from lxml import html
|
||||
from pydantic import ValidationError
|
||||
from saxonche import PySaxonProcessor
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=1,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Create Saxon processor once and reuse across requests.
|
||||
saxon_proc = PySaxonProcessor(license=False)
|
||||
xpath_proc = saxon_proc.new_xpath_processor()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Parse HTML with lxml.
|
||||
parsed_html = html.fromstring(await context.http_response.read())
|
||||
# Convert relative URLs to absolute before extracting links.
|
||||
parsed_html.make_links_absolute(context.request.url, resolve_base_href=True)
|
||||
# Convert parsed HTML to XML for Saxon processing.
|
||||
xml = html.tostring(parsed_html, encoding='unicode', method='xml')
|
||||
# Parse XML with Saxon.
|
||||
parsed_xml = saxon_proc.parse_xml(xml_text=xml)
|
||||
# Set the parsed context for XPath evaluation.
|
||||
xpath_proc.set_context(xdm_item=parsed_xml)
|
||||
|
||||
# Extract data using XPath 2.0 string() function.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': xpath_proc.evaluate_single('.//title/string()'),
|
||||
'h1s': [str(h) for h in (xpath_proc.evaluate('//h1/string()') or [])],
|
||||
'h2s': [str(h) for h in (xpath_proc.evaluate('//h2/string()') or [])],
|
||||
'h3s': [str(h) for h in (xpath_proc.evaluate('//h3/string()') or [])],
|
||||
}
|
||||
await context.push_data(data)
|
||||
|
||||
# XPath 2.0 with distinct-values() to get unique links and remove fragments.
|
||||
links_xpath = """
|
||||
distinct-values(
|
||||
for $href in //a/@href[
|
||||
not(starts-with(., "#"))
|
||||
and not(starts-with(., "javascript:"))
|
||||
and not(starts-with(., "mailto:"))
|
||||
]
|
||||
return replace($href, "#.*$", "")
|
||||
)
|
||||
"""
|
||||
|
||||
extracted_requests = []
|
||||
|
||||
# Extract links.
|
||||
for item in xpath_proc.evaluate(links_xpath) or []:
|
||||
url = item.string_value
|
||||
try:
|
||||
request = Request.from_url(url)
|
||||
except ValidationError as exc:
|
||||
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
|
||||
continue
|
||||
extracted_requests.append(request)
|
||||
|
||||
# Add extracted requests to the queue with the same-domain strategy.
|
||||
await context.add_requests(extracted_requests, strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ParselCrawler instance
|
||||
crawler = ParselCrawler(
|
||||
# Limit the crawl to 10 requests
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
# Define the default request handler
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract data using Parsel's XPath and CSS selectors
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset
|
||||
await context.push_data(data)
|
||||
|
||||
# Enqueue links found on the page for further crawling
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,64 @@
|
||||
import asyncio
|
||||
|
||||
from pydantic import ValidationError
|
||||
from pyquery import PyQuery
|
||||
from yarl import URL
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=1,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Parse the HTML content using PyQuery.
|
||||
parsed_html = PyQuery(await context.http_response.read())
|
||||
|
||||
# Extract data using jQuery-style selectors.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': parsed_html('title').text(),
|
||||
'h1s': [h1.text() for h1 in parsed_html('h1').items()],
|
||||
'h2s': [h2.text() for h2 in parsed_html('h2').items()],
|
||||
'h3s': [h3.text() for h3 in parsed_html('h3').items()],
|
||||
}
|
||||
await context.push_data(data)
|
||||
|
||||
# Css selector to extract valid href attributes.
|
||||
links_selector = (
|
||||
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
|
||||
)
|
||||
base_url = URL(context.request.url)
|
||||
|
||||
extracted_requests = []
|
||||
|
||||
# Extract links.
|
||||
for item in parsed_html(links_selector).items():
|
||||
href = item.attr('href')
|
||||
if not href:
|
||||
continue
|
||||
|
||||
# Convert relative URLs to absolute if needed.
|
||||
url = str(base_url.join(URL(str(href))))
|
||||
try:
|
||||
request = Request.from_url(url)
|
||||
except ValidationError as exc:
|
||||
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
|
||||
continue
|
||||
extracted_requests.append(request)
|
||||
|
||||
# Add extracted requests to the queue with the same-domain strategy.
|
||||
await context.add_requests(extracted_requests, strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,74 @@
|
||||
import asyncio
|
||||
|
||||
from pydantic import ValidationError
|
||||
from scrapling.parser import Selector
|
||||
from yarl import URL
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=1,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Parse the HTML content using Scrapling.
|
||||
page = Selector(await context.http_response.read(), url=context.request.url)
|
||||
|
||||
# Extract data using Xpath selectors with .get_all_text method for full text
|
||||
# content.
|
||||
title_el = page.xpath_first('//title')
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': title_el.text if isinstance(title_el, Selector) else title_el,
|
||||
'h1s': [
|
||||
h1.get_all_text() if isinstance(h1, Selector) else h1
|
||||
for h1 in page.xpath('//h1')
|
||||
],
|
||||
'h2s': [
|
||||
h2.get_all_text() if isinstance(h2, Selector) else h2
|
||||
for h2 in page.xpath('//h2')
|
||||
],
|
||||
'h3s': [
|
||||
h3.get_all_text() if isinstance(h3, Selector) else h3
|
||||
for h3 in page.xpath('//h3')
|
||||
],
|
||||
}
|
||||
await context.push_data(data)
|
||||
|
||||
# Css selector to extract valid href attributes.
|
||||
links_selector = (
|
||||
'a[href]:not([href^="#"]):not([href^="javascript:"]):not([href^="mailto:"])'
|
||||
)
|
||||
base_url = URL(context.request.url)
|
||||
extracted_requests = []
|
||||
|
||||
# Extract links.
|
||||
for item in page.css(links_selector):
|
||||
href = item.attrib.get('href') if isinstance(item, Selector) else None
|
||||
if not href:
|
||||
continue
|
||||
|
||||
# Convert relative URLs to absolute if needed.
|
||||
url = str(base_url.join(URL(href)))
|
||||
try:
|
||||
request = Request.from_url(url)
|
||||
except ValidationError as exc:
|
||||
context.log.warning(f'Skipping invalid URL "{url}": {exc}')
|
||||
continue
|
||||
extracted_requests.append(request)
|
||||
|
||||
# Add extracted requests to the queue with the same-domain strategy.
|
||||
await context.add_requests(extracted_requests, strategy='same-domain')
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,34 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
AdaptivePlaywrightCrawler,
|
||||
AdaptivePlaywrightCrawlingContext,
|
||||
)
|
||||
|
||||
from .selectolax_parser import SelectolaxLexborParser
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler: AdaptivePlaywrightCrawler = AdaptivePlaywrightCrawler(
|
||||
max_requests_per_crawl=10,
|
||||
# Use custom Selectolax parser for static content parsing.
|
||||
static_parser=SelectolaxLexborParser(),
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handle_request(context: AdaptivePlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.query_selector_one('title'),
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
from dataclasses import dataclass, fields
|
||||
|
||||
from selectolax.lexbor import LexborHTMLParser
|
||||
from typing_extensions import Self
|
||||
|
||||
from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext
|
||||
|
||||
|
||||
# Custom context for Selectolax parser, you can add your own methods here
|
||||
# to facilitate working with the parsed document.
|
||||
@dataclass(frozen=True)
|
||||
class SelectolaxLexborContext(ParsedHttpCrawlingContext[LexborHTMLParser]):
|
||||
"""Crawling context providing access to the parsed page.
|
||||
|
||||
This context is passed to request handlers and includes all standard
|
||||
context methods (push_data, enqueue_links, etc.) plus custom helpers.
|
||||
"""
|
||||
|
||||
@property
|
||||
def parser(self) -> LexborHTMLParser:
|
||||
"""Convenient alias for accessing the parsed document."""
|
||||
return self.parsed_content
|
||||
|
||||
@classmethod
|
||||
def from_parsed_http_crawling_context(
|
||||
cls, context: ParsedHttpCrawlingContext[LexborHTMLParser]
|
||||
) -> Self:
|
||||
"""Create custom context from the base context.
|
||||
|
||||
Copies all fields from the base context to preserve framework
|
||||
functionality while adding custom interface.
|
||||
"""
|
||||
return cls(
|
||||
**{field.name: getattr(context, field.name) for field in fields(context)}
|
||||
)
|
||||
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from selectolax.lexbor import LexborHTMLParser, LexborNode
|
||||
|
||||
from crawlee.crawlers import AbstractHttpCrawler, HttpCrawlerOptions
|
||||
|
||||
from .selectolax_context import SelectolaxLexborContext
|
||||
from .selectolax_parser import SelectolaxLexborParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from typing_extensions import Unpack
|
||||
|
||||
from crawlee.crawlers._abstract_http import ParsedHttpCrawlingContext
|
||||
|
||||
|
||||
# Custom crawler using custom context, It is optional and you can use
|
||||
# AbstractHttpCrawler directly with SelectolaxLexborParser if you don't need
|
||||
# any custom context methods.
|
||||
class SelectolaxLexborCrawler(
|
||||
AbstractHttpCrawler[SelectolaxLexborContext, LexborHTMLParser, LexborNode]
|
||||
):
|
||||
"""Custom crawler using Selectolax Lexbor for HTML parsing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
**kwargs: Unpack[HttpCrawlerOptions[SelectolaxLexborContext]],
|
||||
) -> None:
|
||||
# Final step converts the base context to custom context type.
|
||||
async def final_step(
|
||||
context: ParsedHttpCrawlingContext[LexborHTMLParser],
|
||||
) -> AsyncGenerator[SelectolaxLexborContext, None]:
|
||||
# Yield custom context wrapping with additional functionality around the base
|
||||
# context.
|
||||
yield SelectolaxLexborContext.from_parsed_http_crawling_context(context)
|
||||
|
||||
# Build context pipeline: HTTP request -> parsing -> custom context.
|
||||
kwargs['_context_pipeline'] = (
|
||||
self._create_static_content_crawler_pipeline().compose(final_step)
|
||||
)
|
||||
super().__init__(
|
||||
parser=SelectolaxLexborParser(),
|
||||
**kwargs,
|
||||
)
|
||||
@@ -0,0 +1,27 @@
|
||||
import asyncio
|
||||
|
||||
from .selectolax_crawler import SelectolaxLexborContext, SelectolaxLexborCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = SelectolaxLexborCrawler(
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handle_request(context: SelectolaxLexborContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.parser.css_first('title').text(),
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from selectolax.lexbor import LexborHTMLParser, LexborNode
|
||||
from typing_extensions import override
|
||||
|
||||
from crawlee.crawlers._abstract_http import AbstractHttpParser
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable, Sequence
|
||||
|
||||
from crawlee.http_clients import HttpResponse
|
||||
|
||||
|
||||
class SelectolaxLexborParser(AbstractHttpParser[LexborHTMLParser, LexborNode]):
|
||||
"""Parser for parsing HTTP response using Selectolax Lexbor."""
|
||||
|
||||
@override
|
||||
async def parse(self, response: HttpResponse) -> LexborHTMLParser:
|
||||
"""Parse HTTP response body into a document object."""
|
||||
response_body = await response.read()
|
||||
# Run parsing in a thread to avoid blocking the event loop.
|
||||
return await asyncio.to_thread(LexborHTMLParser, response_body)
|
||||
|
||||
@override
|
||||
async def parse_text(self, text: str) -> LexborHTMLParser:
|
||||
"""Parse raw HTML string into a document object."""
|
||||
return LexborHTMLParser(text)
|
||||
|
||||
@override
|
||||
async def select(
|
||||
self, parsed_content: LexborHTMLParser, selector: str
|
||||
) -> Sequence[LexborNode]:
|
||||
"""Select elements matching a CSS selector."""
|
||||
return tuple(item for item in parsed_content.css(selector))
|
||||
|
||||
@override
|
||||
def is_matching_selector(
|
||||
self, parsed_content: LexborHTMLParser, selector: str
|
||||
) -> bool:
|
||||
"""Check if any element matches the selector."""
|
||||
return parsed_content.css_first(selector) is not None
|
||||
|
||||
@override
|
||||
def find_links(
|
||||
self, parsed_content: LexborHTMLParser, selector: str, attribute: str
|
||||
) -> Iterable[str]:
|
||||
"""Extract href attributes from elements matching the selector.
|
||||
|
||||
Used by `enqueue_links` helper to discover URLs.
|
||||
"""
|
||||
link: LexborNode
|
||||
urls: list[str] = []
|
||||
for link in parsed_content.css(selector):
|
||||
url = link.attributes.get(attribute)
|
||||
if url:
|
||||
urls.append(url.strip())
|
||||
return urls
|
||||
@@ -0,0 +1,85 @@
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from crawlee import ConcurrencySettings, Request
|
||||
from crawlee.crawlers import (
|
||||
HttpCrawler,
|
||||
HttpCrawlingContext,
|
||||
)
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_requests_per_crawl=10,
|
||||
# Configure to use a single persistent session throughout the crawl
|
||||
max_session_rotations=0,
|
||||
# Limit request rate to avoid triggering anti-scraping measures
|
||||
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
|
||||
session_pool=SessionPool(
|
||||
max_pool_size=1,
|
||||
create_session_settings={
|
||||
# Set high value to ensure the session isn't replaced during crawling
|
||||
'max_usage_count': 999_999,
|
||||
# Set high value to prevent session expiration during crawling
|
||||
'max_age': timedelta(hours=999_999),
|
||||
# Higher error tolerance before the session is considered blocked
|
||||
# Make sure you implement proper error handling in your code
|
||||
'max_error_score': 100,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
# Default request handler for normal page processing
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Specialized handler for the login API request
|
||||
@crawler.router.handler('login')
|
||||
async def login_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing login at {context.request.url} ...')
|
||||
|
||||
# Verify that a session is available before proceeding
|
||||
if not context.session:
|
||||
raise RuntimeError('Session not found')
|
||||
|
||||
# Parse the API response containing authentication tokens and user data
|
||||
data = json.loads(await context.http_response.read())
|
||||
|
||||
# Extract authentication data from the response
|
||||
token = data['token']
|
||||
expires = data['expires'].replace('Z', '+00:00')
|
||||
expires_int = int(datetime.fromisoformat(expires).timestamp())
|
||||
user_id = data['userId']
|
||||
username = data['username']
|
||||
|
||||
# Set authentication cookies in the session that will be used
|
||||
# for subsequent requests
|
||||
context.session.cookies.set(name='token', value=token, expires=expires_int)
|
||||
context.session.cookies.set(name='userID', value=user_id)
|
||||
context.session.cookies.set(name='userName', value=username)
|
||||
|
||||
# After successful authentication, continue crawling with the
|
||||
# authenticated session
|
||||
await context.add_requests(['https://demoqa.com/BookStore/v1/Books'])
|
||||
|
||||
# Create a POST request to the authentication API endpoint
|
||||
# This will trigger the login_handler when executed
|
||||
request = Request.from_url(
|
||||
'https://demoqa.com/Account/v1/Login',
|
||||
label='login',
|
||||
method='POST',
|
||||
payload=json.dumps(
|
||||
{'userName': 'crawlee_test', 'password': 'Test1234!'}
|
||||
).encode(),
|
||||
headers={'Content-Type': 'application/json'},
|
||||
)
|
||||
|
||||
# Start the crawling process with the login request
|
||||
await crawler.run([request])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,70 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee import ConcurrencySettings, Request
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
)
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
max_requests_per_crawl=10,
|
||||
headless=True,
|
||||
browser_type='chromium',
|
||||
# We only have one session and it shouldn't rotate
|
||||
max_session_rotations=0,
|
||||
# Limit crawling intensity to avoid blocking
|
||||
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
|
||||
session_pool=SessionPool(
|
||||
# Limit the pool to one session
|
||||
max_pool_size=1,
|
||||
create_session_settings={
|
||||
# High value for session usage limit
|
||||
'max_usage_count': 999_999,
|
||||
# High value for session lifetime
|
||||
'max_age': timedelta(hours=999_999),
|
||||
# High score allows the session to encounter more errors
|
||||
# before crawlee decides the session is blocked
|
||||
# Make sure you know how to handle these errors
|
||||
'max_error_score': 100,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
# The main handler for processing requests
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# A handler for the login page
|
||||
@crawler.router.handler('login')
|
||||
async def login_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing login {context.request.url} ...')
|
||||
|
||||
# Check if the session is available
|
||||
if not context.session:
|
||||
raise RuntimeError('Session not found')
|
||||
|
||||
# Entering data into the form, `delay` to simulate human typing
|
||||
# Without this, the data will be entered instantly
|
||||
await context.page.type('#userName', 'crawlee_test', delay=100)
|
||||
await context.page.type('#password', 'Test1234!', delay=100)
|
||||
await context.page.click('#login', delay=100)
|
||||
|
||||
# Wait for an element confirming that we have successfully
|
||||
# logged in to the site
|
||||
await context.page.locator('#userName-value').first.wait_for(state='visible')
|
||||
context.log.info('Login successful!')
|
||||
|
||||
# Moving on to the basic flow of crawling
|
||||
await context.add_requests(['https://demoqa.com/books'])
|
||||
|
||||
# We start crawling with login. This is necessary to access the rest of the pages
|
||||
await crawler.run([Request.from_url('https://demoqa.com/login', label='login')])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
headless=False,
|
||||
browser_type='chromium',
|
||||
# Browser launch options
|
||||
browser_launch_options={
|
||||
# For support `msedge` channel you need to install it
|
||||
# `playwright install msedge`
|
||||
'channel': 'msedge',
|
||||
'slow_mo': 200,
|
||||
},
|
||||
# Context launch options, applied to each page as it is created
|
||||
browser_new_context_options={
|
||||
'color_scheme': 'dark',
|
||||
# Set headers
|
||||
'extra_http_headers': {
|
||||
'Custom-Header': 'my-header',
|
||||
'Accept-Language': 'en',
|
||||
},
|
||||
# Set only User Agent
|
||||
'user_agent': 'My-User-Agent',
|
||||
},
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crawlee.browsers import BrowserPool
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crawlee.browsers._browser_controller import BrowserController
|
||||
from crawlee.browsers._browser_plugin import BrowserPlugin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with BrowserPool() as browser_pool:
|
||||
|
||||
@browser_pool.pre_launch_hook
|
||||
async def log_browser_launch(page_id: str, plugin: BrowserPlugin) -> None:
|
||||
"""Log before a new browser instance is launched."""
|
||||
logger.info(f'Launching {plugin.browser_type} browser for page {page_id}...')
|
||||
|
||||
@browser_pool.post_launch_hook
|
||||
async def log_browser_launched(
|
||||
page_id: str, controller: BrowserController
|
||||
) -> None:
|
||||
"""Log after a new browser instance has been launched."""
|
||||
logger.info(f'Browser launched for page {page_id}, controller: {controller}')
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=browser_pool,
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from crawlee.browsers import BrowserPool
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storages import KeyValueStore
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crawlee.browsers._browser_controller import BrowserController
|
||||
from crawlee.browsers._types import CrawleePage
|
||||
from crawlee.proxy_configuration import ProxyInfo
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with BrowserPool() as browser_pool:
|
||||
|
||||
@browser_pool.pre_page_create_hook
|
||||
async def log_page_init(
|
||||
page_id: str,
|
||||
_browser_controller: BrowserController,
|
||||
_browser_new_context_options: dict[str, Any],
|
||||
_proxy_info: ProxyInfo | None,
|
||||
) -> None:
|
||||
"""Log when a new page is about to be created."""
|
||||
logger.info(f'Creating page {page_id}...')
|
||||
|
||||
@browser_pool.post_page_create_hook
|
||||
async def set_viewport(
|
||||
crawlee_page: CrawleePage, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Set a fixed viewport size on each newly created page."""
|
||||
await crawlee_page.page.set_viewport_size({'width': 1280, 'height': 1024})
|
||||
|
||||
@browser_pool.pre_page_close_hook
|
||||
async def save_screenshot(
|
||||
crawlee_page: CrawleePage, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Save a screenshot to KeyValueStore before each page is closed."""
|
||||
kvs = await KeyValueStore.open()
|
||||
|
||||
screenshot = await crawlee_page.page.screenshot()
|
||||
await kvs.set_value(
|
||||
key=f'screenshot-{crawlee_page.id}',
|
||||
value=screenshot,
|
||||
content_type='image/png',
|
||||
)
|
||||
logger.info(f'Saved screenshot for page {crawlee_page.id}.')
|
||||
|
||||
@browser_pool.post_page_close_hook
|
||||
async def log_page_closed(
|
||||
page_id: str, _browser_controller: BrowserController
|
||||
) -> None:
|
||||
"""Log after each page is closed."""
|
||||
logger.info(f'Page {page_id} closed successfully.')
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=browser_pool,
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,38 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a plugin for each required browser.
|
||||
plugin_chromium = PlaywrightBrowserPlugin(
|
||||
browser_type='chromium', max_open_pages_per_browser=1
|
||||
)
|
||||
plugin_firefox = PlaywrightBrowserPlugin(
|
||||
browser_type='firefox', max_open_pages_per_browser=1
|
||||
)
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=BrowserPool(plugins=[plugin_chromium, plugin_firefox]),
|
||||
# Limit the crawl to max requests. Remove or increase it for crawling all links.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
browser_name = (
|
||||
context.page.context.browser.browser_type.name
|
||||
if context.page.context.browser
|
||||
else 'undefined'
|
||||
)
|
||||
context.log.info(f'Processing {context.request.url} with {browser_name} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev', 'https://apify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPostNavCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
from crawlee.errors import SessionError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def configure_page(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
context.log.info(f'Navigating to {context.request.url} ...')
|
||||
|
||||
# block stylesheets, images, fonts and other static assets
|
||||
# to speed up page loading
|
||||
await context.block_requests()
|
||||
|
||||
@crawler.post_navigation_hook
|
||||
async def custom_captcha_check(context: PlaywrightPostNavCrawlingContext) -> None:
|
||||
# check if the page contains a captcha
|
||||
captcha_element = context.page.locator('input[name="captcha"]').first
|
||||
if await captcha_element.is_visible():
|
||||
context.log.warning('Captcha detected! Skipping the page.')
|
||||
raise SessionError('Captcha detected')
|
||||
|
||||
# Run the crawler with the initial list of URLs.
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,35 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.browsers import BrowserPool, PlaywrightBrowserPlugin
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
browser_pool=BrowserPool(
|
||||
plugins=[
|
||||
PlaywrightBrowserPlugin(
|
||||
browser_type='chromium',
|
||||
browser_launch_options={
|
||||
'headless': False,
|
||||
'channel': 'msedge',
|
||||
'slow_mo': 200,
|
||||
},
|
||||
browser_new_context_options={
|
||||
'color_scheme': 'dark',
|
||||
'extra_http_headers': {
|
||||
'Custom-Header': 'my-header',
|
||||
'Accept-Language': 'en',
|
||||
},
|
||||
'user_agent': 'My-User-Agent',
|
||||
},
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee.crawlers import AdaptivePlaywrightCrawler, AdaptivePlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
|
||||
# Locate element h2 within 5 seconds
|
||||
h2 = await context.query_selector_one('h2', timedelta(milliseconds=5000))
|
||||
# Do stuff with element found by the selector
|
||||
context.log.info(h2)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import AdaptivePlaywrightCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
|
||||
# Arguments relevant only for PlaywrightCrawler
|
||||
playwright_crawler_specific_kwargs={
|
||||
'headless': False,
|
||||
'browser_type': 'chromium',
|
||||
},
|
||||
# Common arguments relevant to all crawlers
|
||||
max_crawl_depth=5,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import AdaptivePlaywrightCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
|
||||
# Arguments relevant only for PlaywrightCrawler
|
||||
playwright_crawler_specific_kwargs={
|
||||
'headless': False,
|
||||
'browser_type': 'chromium',
|
||||
},
|
||||
# Common arguments relevant to all crawlers
|
||||
max_crawl_depth=5,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee._types import RequestHandlerRunResult
|
||||
from crawlee.crawlers import (
|
||||
AdaptivePlaywrightCrawler,
|
||||
RenderingType,
|
||||
RenderingTypePrediction,
|
||||
RenderingTypePredictor,
|
||||
)
|
||||
|
||||
|
||||
class CustomRenderingTypePredictor(RenderingTypePredictor):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
|
||||
self._learning_data = list[tuple[Request, RenderingType]]()
|
||||
|
||||
def predict(self, request: Request) -> RenderingTypePrediction:
|
||||
# Some custom logic that produces some `RenderingTypePrediction`
|
||||
# based on the `request` input.
|
||||
rendering_type: RenderingType = (
|
||||
'static' if 'abc' in request.url else 'client only'
|
||||
)
|
||||
|
||||
return RenderingTypePrediction(
|
||||
# Recommends `static` rendering type -> HTTP-based sub crawler will be used.
|
||||
rendering_type=rendering_type,
|
||||
# Recommends that both sub crawlers should run with 20% chance. When both sub
|
||||
# crawlers are running, the predictor can compare results and learn.
|
||||
# High number means that predictor is not very confident about the
|
||||
# `rendering_type`, low number means that predictor is very confident.
|
||||
detection_probability_recommendation=0.2,
|
||||
)
|
||||
|
||||
def store_result(self, request: Request, rendering_type: RenderingType) -> None:
|
||||
# This function allows predictor to store new learning data and retrain itself
|
||||
# if needed. `request` is input for prediction and `rendering_type` is the correct
|
||||
# prediction.
|
||||
self._learning_data.append((request, rendering_type))
|
||||
# retrain
|
||||
|
||||
|
||||
def result_checker(result: RequestHandlerRunResult) -> bool:
|
||||
# Some function that inspects produced `result` and returns `True` if the result
|
||||
# is correct.
|
||||
return bool(result) # Check something on result
|
||||
|
||||
|
||||
def result_comparator(
|
||||
result_1: RequestHandlerRunResult, result_2: RequestHandlerRunResult
|
||||
) -> bool:
|
||||
# Some function that inspects two results and returns `True` if they are
|
||||
# considered equivalent. It is used when comparing results produced by HTTP-based
|
||||
# sub crawler and playwright based sub crawler.
|
||||
return (
|
||||
result_1.push_data_calls == result_2.push_data_calls
|
||||
) # For example compare `push_data` calls.
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_parsel_static_parser(
|
||||
rendering_type_predictor=CustomRenderingTypePredictor(),
|
||||
result_checker=result_checker,
|
||||
result_comparator=result_comparator,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
|
||||
from playwright.async_api import Route
|
||||
|
||||
from crawlee.crawlers import (
|
||||
AdaptivePlaywrightCrawler,
|
||||
AdaptivePlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser()
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def hook(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
"""Hook executed both in static sub crawler and playwright sub crawler.
|
||||
|
||||
Trying to access `context.page` in this hook would raise `AdaptiveContextError`
|
||||
for pages crawled without playwright.
|
||||
"""
|
||||
context.log.info(f'pre navigation hook for: {context.request.url}')
|
||||
|
||||
@crawler.pre_navigation_hook(playwright_only=True)
|
||||
async def hook_playwright(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
"""Hook executed only in playwright sub crawler."""
|
||||
|
||||
async def some_routing_function(route: Route) -> None:
|
||||
await route.continue_()
|
||||
|
||||
await context.page.route('*/**', some_routing_function)
|
||||
context.log.info(
|
||||
f'Playwright only pre navigation hook for: {context.request.url}'
|
||||
)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
context.log.info(f'Extracted data: {data}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.page.title(),
|
||||
}
|
||||
context.log.info(f'Extracted data: {data}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
|
||||
# The proxy URLs are rotated in a round-robin.
|
||||
proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/
|
||||
proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/
|
||||
proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
use_session_pool=True,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
use_session_pool=True,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
tiered_proxy_urls=[
|
||||
# No proxy tier.
|
||||
# Optional in case you do not want to use any proxy on lowest tier.
|
||||
[None],
|
||||
# lower tier, cheaper, preferred as long as they work
|
||||
[
|
||||
'http://cheap-datacenter-proxy-1.com/',
|
||||
'http://cheap-datacenter-proxy-2.com/',
|
||||
],
|
||||
# higher tier, more expensive, used as a fallback
|
||||
[
|
||||
'http://expensive-residential-proxy-1.com/',
|
||||
'http://expensive-residential-proxy-2.com/',
|
||||
],
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
tiered_proxy_urls=[
|
||||
# No proxy tier.
|
||||
# Optional in case you do not want to use any proxy on lowest tier.
|
||||
[None],
|
||||
# lower tier, cheaper, preferred as long as they work
|
||||
[
|
||||
'http://cheap-datacenter-proxy-1.com/',
|
||||
'http://cheap-datacenter-proxy-2.com/',
|
||||
],
|
||||
# higher tier, more expensive, used as a fallback
|
||||
[
|
||||
'http://expensive-residential-proxy-1.com/',
|
||||
'http://expensive-residential-proxy-2.com/',
|
||||
],
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.request_loaders import RequestList
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Open the request list, if it does not exist, it will be created.
|
||||
# Leave name empty to use the default request list.
|
||||
request_list = RequestList(
|
||||
name='my-request-list',
|
||||
requests=[
|
||||
'https://apify.com/',
|
||||
'https://crawlee.dev/',
|
||||
'https://crawlee.dev/python/',
|
||||
],
|
||||
)
|
||||
|
||||
# Fetch and process requests from the queue.
|
||||
while request := await request_list.fetch_next_request():
|
||||
# Do something with it...
|
||||
print(f'Processing {request.url}')
|
||||
|
||||
# And mark it as handled.
|
||||
await request_list.mark_request_as_handled(request)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.request_loaders import RequestList
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Disable clearing the `KeyValueStore` on each run.
|
||||
# This is necessary so that the state keys are not cleared at startup.
|
||||
# The recommended way to achieve this behavior is setting the environment variable
|
||||
# `CRAWLEE_PURGE_ON_START=0`
|
||||
configuration = service_locator.get_configuration()
|
||||
configuration.purge_on_start = False
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Open the request list, if it does not exist, it will be created.
|
||||
# Leave name empty to use the default request list.
|
||||
request_list = RequestList(
|
||||
name='my-request-list',
|
||||
requests=[
|
||||
'https://apify.com/',
|
||||
'https://crawlee.dev/',
|
||||
'https://crawlee.dev/python/',
|
||||
],
|
||||
# Enable persistence
|
||||
persist_state_key='my-persist-state',
|
||||
persist_requests_key='my-persist-requests',
|
||||
)
|
||||
|
||||
# We receive only one request.
|
||||
# Each time you run it, it will be a new request until you exhaust the `RequestList`.
|
||||
request = await request_list.fetch_next_request()
|
||||
if request:
|
||||
logger.info(f'Processing request: {request.url}')
|
||||
# Do something with it...
|
||||
|
||||
# And mark it as handled.
|
||||
await request_list.mark_request_as_handled(request)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.request_loaders import RequestList
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a static request list.
|
||||
request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
|
||||
|
||||
# highlight-start
|
||||
# Convert the request list to a request manager using the to_tandem method.
|
||||
# It is a tandem with the default request queue.
|
||||
request_manager = await request_list.to_tandem()
|
||||
# highlight-end
|
||||
|
||||
# Create a crawler and pass the request manager to it.
|
||||
crawler = ParselCrawler(
|
||||
request_manager=request_manager,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# New links will be enqueued directly to the queue.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data using Parsel's XPath and CSS selectors.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.request_loaders import RequestList, RequestManagerTandem
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a static request list.
|
||||
request_list = RequestList(['https://crawlee.dev', 'https://apify.com'])
|
||||
|
||||
# Open the default request queue.
|
||||
request_queue = await RequestQueue.open()
|
||||
|
||||
# And combine them together to a single request manager.
|
||||
request_manager = RequestManagerTandem(request_list, request_queue)
|
||||
|
||||
# Create a crawler and pass the request manager to it.
|
||||
crawler = ParselCrawler(
|
||||
request_manager=request_manager,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# New links will be enqueued directly to the queue.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data using Parsel's XPath and CSS selectors.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
from crawlee.request_loaders import SitemapRequestLoader
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an HTTP client for fetching the sitemap.
|
||||
http_client = ImpitHttpClient()
|
||||
|
||||
# Create a sitemap request loader with filtering rules.
|
||||
sitemap_loader = SitemapRequestLoader(
|
||||
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
|
||||
http_client=http_client,
|
||||
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
|
||||
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
|
||||
)
|
||||
|
||||
# We work with the loader until we process all relevant links from the sitemap.
|
||||
while request := await sitemap_loader.fetch_next_request():
|
||||
# Do something with it...
|
||||
print(f'Processing {request.url}')
|
||||
|
||||
# And mark it as handled.
|
||||
await sitemap_loader.mark_request_as_handled(request)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,45 @@
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
from crawlee.request_loaders import SitemapRequestLoader
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s-%(levelname)s-%(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# Disable clearing the `KeyValueStore` on each run.
|
||||
# This is necessary so that the state keys are not cleared at startup.
|
||||
# The recommended way to achieve this behavior is setting the environment variable
|
||||
# `CRAWLEE_PURGE_ON_START=0`
|
||||
configuration = service_locator.get_configuration()
|
||||
configuration.purge_on_start = False
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an HTTP client for fetching sitemaps
|
||||
# Use the context manager for `SitemapRequestLoader` to correctly save the state when
|
||||
# the work is completed.
|
||||
async with (
|
||||
ImpitHttpClient() as http_client,
|
||||
SitemapRequestLoader(
|
||||
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
|
||||
http_client=http_client,
|
||||
# Enable persistence
|
||||
persist_state_key='my-persist-state',
|
||||
) as sitemap_loader,
|
||||
):
|
||||
# We receive only one request.
|
||||
# Each time you run it, it will be a new request until you exhaust the sitemap.
|
||||
request = await sitemap_loader.fetch_next_request()
|
||||
if request:
|
||||
logger.info(f'Processing request: {request.url}')
|
||||
# Do something with it...
|
||||
|
||||
# And mark it as handled.
|
||||
await sitemap_loader.mark_request_as_handled(request)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,53 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
from crawlee.request_loaders import SitemapRequestLoader
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an HTTP client for fetching the sitemap.
|
||||
http_client = ImpitHttpClient()
|
||||
|
||||
# Create a sitemap request loader with filtering rules.
|
||||
sitemap_loader = SitemapRequestLoader(
|
||||
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
|
||||
http_client=http_client,
|
||||
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
|
||||
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
|
||||
)
|
||||
|
||||
# highlight-start
|
||||
# Convert the sitemap loader into a request manager linked
|
||||
# to the default request queue.
|
||||
request_manager = await sitemap_loader.to_tandem()
|
||||
# highlight-end
|
||||
|
||||
# Create a crawler and pass the request manager to it.
|
||||
crawler = ParselCrawler(
|
||||
request_manager=request_manager,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# New links will be enqueued directly to the queue.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data using Parsel's XPath and CSS selectors.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.http_clients import ImpitHttpClient
|
||||
from crawlee.request_loaders import RequestManagerTandem, SitemapRequestLoader
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create an HTTP client for fetching the sitemap.
|
||||
http_client = ImpitHttpClient()
|
||||
|
||||
# Create a sitemap request loader with filtering rules.
|
||||
sitemap_loader = SitemapRequestLoader(
|
||||
sitemap_urls=['https://crawlee.dev/sitemap.xml'],
|
||||
http_client=http_client,
|
||||
include=[re.compile(r'.*docs.*')], # Only include URLs containing 'docs'.
|
||||
max_buffer_size=500, # Keep up to 500 URLs in memory before processing.
|
||||
)
|
||||
|
||||
# Open the default request queue.
|
||||
request_queue = await RequestQueue.open()
|
||||
|
||||
# And combine them together to a single request manager.
|
||||
request_manager = RequestManagerTandem(sitemap_loader, request_queue)
|
||||
|
||||
# Create a crawler and pass the request manager to it.
|
||||
crawler = ParselCrawler(
|
||||
request_manager=request_manager,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# New links will be enqueued directly to the queue.
|
||||
await context.enqueue_links()
|
||||
|
||||
# Extract data using Parsel's XPath and CSS selectors.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.selector.xpath('//title/text()').get(),
|
||||
}
|
||||
|
||||
# Push extracted data to the dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,51 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import HttpHeaders
|
||||
from crawlee.crawlers import (
|
||||
AdaptivePlaywrightCrawler,
|
||||
AdaptivePlaywrightCrawlingContext,
|
||||
AdaptivePlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def common_setup(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
# Common pre-navigation hook - runs for both HTTP and browser requests.
|
||||
context.request.headers |= HttpHeaders(
|
||||
{'Accept': 'text/html,application/xhtml+xml'},
|
||||
)
|
||||
|
||||
@crawler.pre_navigation_hook(playwright_only=True)
|
||||
async def browser_setup(context: AdaptivePlaywrightPreNavCrawlingContext) -> None:
|
||||
# Playwright-specific pre-navigation hook - runs only when browser is used.
|
||||
await context.page.set_viewport_size({'width': 1280, 'height': 720})
|
||||
if context.block_requests:
|
||||
await context.block_requests(extra_url_patterns=['*.css', '*.js'])
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: AdaptivePlaywrightCrawlingContext) -> None:
|
||||
# Extract title using the unified context interface.
|
||||
title_tag = context.parsed_content.find('title')
|
||||
title = title_tag.get_text() if title_tag else None
|
||||
|
||||
# Extract other data consistently across both modes.
|
||||
links = [a.get('href') for a in context.parsed_content.find_all('a', href=True)]
|
||||
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
'links': links,
|
||||
}
|
||||
)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,92 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.router import Router
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a custom router instance
|
||||
router = Router[ParselCrawlingContext]()
|
||||
|
||||
# Define the default handler (fallback for requests without specific labels)
|
||||
@router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing homepage: {context.request.url}')
|
||||
|
||||
# Extract page title
|
||||
title = context.selector.css('title::text').get() or 'No title found'
|
||||
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
'page_type': 'homepage',
|
||||
}
|
||||
)
|
||||
|
||||
# Find and enqueue collection/category links
|
||||
await context.enqueue_links(selector='a[href*="/collections/"]', label='CATEGORY')
|
||||
|
||||
# Define a handler for category pages
|
||||
@router.handler('CATEGORY')
|
||||
async def category_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing category page: {context.request.url}')
|
||||
|
||||
# Extract category information
|
||||
category_title = context.selector.css('h1::text').get() or 'Unknown Category'
|
||||
product_count = len(context.selector.css('.product-item').getall())
|
||||
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'type': 'category',
|
||||
'category_title': category_title,
|
||||
'product_count': product_count,
|
||||
'handler': 'category',
|
||||
}
|
||||
)
|
||||
|
||||
# Enqueue product links from this category
|
||||
await context.enqueue_links(selector='a[href*="/products/"]', label='PRODUCT')
|
||||
|
||||
# Define a handler for product detail pages
|
||||
@router.handler('PRODUCT')
|
||||
async def product_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing product page: {context.request.url}')
|
||||
|
||||
# Extract detailed product information
|
||||
product_data = {
|
||||
'url': context.request.url,
|
||||
'name': context.selector.css('h1::text').get(),
|
||||
'price': context.selector.css('.price::text').get(),
|
||||
'description': context.selector.css('.product-description p::text').get(),
|
||||
'images': context.selector.css('.product-gallery img::attr(src)').getall(),
|
||||
'in_stock': bool(context.selector.css('.add-to-cart-button').get()),
|
||||
'handler': 'product',
|
||||
}
|
||||
|
||||
await context.push_data(product_data)
|
||||
|
||||
# Create crawler with the router
|
||||
crawler = ParselCrawler(
|
||||
request_handler=router,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
# Start crawling with some initial requests
|
||||
await crawler.run(
|
||||
[
|
||||
# Will use default handler
|
||||
'https://warehouse-theme-metal.myshopify.com/',
|
||||
# Will use category handler
|
||||
Request.from_url(
|
||||
'https://warehouse-theme-metal.myshopify.com/collections/all',
|
||||
label='CATEGORY',
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.router import Router
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a custom router instance
|
||||
router = Router[ParselCrawlingContext]()
|
||||
|
||||
# Define only a default handler
|
||||
@router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract page title
|
||||
title = context.selector.css('title::text').get() or 'No title found'
|
||||
|
||||
# Extract and save basic page data
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
}
|
||||
)
|
||||
|
||||
# Find and enqueue product links for further crawling
|
||||
await context.enqueue_links(
|
||||
selector='a[href*="/products/"]',
|
||||
label='PRODUCT', # Note: no handler for this label, will use default
|
||||
)
|
||||
|
||||
# Create crawler with the custom router
|
||||
crawler = ParselCrawler(
|
||||
request_handler=router,
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
# Start crawling
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,62 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.errors import HttpStatusCodeError
|
||||
|
||||
# HTTP status code constants
|
||||
TOO_MANY_REQUESTS = 429
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a crawler instance
|
||||
crawler = ParselCrawler(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract product information (might fail for some pages)
|
||||
product_name = context.selector.css('h1[data-testid="product-title"]::text').get()
|
||||
if not product_name:
|
||||
raise ValueError('Product name not found - might be a non-product page')
|
||||
|
||||
price = context.selector.css('.price::text').get()
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'product_name': product_name,
|
||||
'price': price,
|
||||
}
|
||||
)
|
||||
|
||||
# Error handler - called when an error occurs during request processing
|
||||
@crawler.error_handler
|
||||
async def error_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
error_name = type(error).__name__
|
||||
context.log.warning(f'Error occurred for {context.request.url}: {error_name}')
|
||||
|
||||
# You can modify the request or context here before retry
|
||||
if (
|
||||
isinstance(error, HttpStatusCodeError)
|
||||
and error.status_code == TOO_MANY_REQUESTS
|
||||
):
|
||||
context.log.info('Rate limited - will retry with delay')
|
||||
# You could modify headers, add delay, etc.
|
||||
elif isinstance(error, ValueError):
|
||||
context.log.info('Parse error - marking request as no retry')
|
||||
context.request.no_retry = True
|
||||
|
||||
# Start crawling
|
||||
await crawler.run(
|
||||
[
|
||||
'https://warehouse-theme-metal.myshopify.com/products/on-running-cloudmonster-2-mens',
|
||||
# Might cause parse error
|
||||
'https://warehouse-theme-metal.myshopify.com/collections/mens-running',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,64 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a crawler instance with retry settings
|
||||
crawler = ParselCrawler(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
max_request_retries=2, # Allow 2 retries before failing
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract product information
|
||||
product_name = context.selector.css('h1[data-testid="product-title"]::text').get()
|
||||
if not product_name:
|
||||
product_name = context.selector.css('h1::text').get() or 'Unknown Product'
|
||||
|
||||
price = context.selector.css('.price::text').get() or 'Price not available'
|
||||
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'product_name': product_name,
|
||||
'price': price,
|
||||
'status': 'success',
|
||||
}
|
||||
)
|
||||
|
||||
# Failed request handler - called when request has exhausted all retries
|
||||
@crawler.failed_request_handler
|
||||
async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
context.log.error(
|
||||
f'Failed to process {context.request.url} after all retries: {error}'
|
||||
)
|
||||
|
||||
# Save failed request information for analysis
|
||||
await context.push_data(
|
||||
{
|
||||
'failed_url': context.request.url,
|
||||
'label': context.request.label,
|
||||
'error_type': type(error).__name__,
|
||||
'error_message': str(error),
|
||||
'retry_count': context.request.retry_count,
|
||||
'status': 'failed',
|
||||
}
|
||||
)
|
||||
|
||||
# Start crawling with some URLs that might fail
|
||||
await crawler.run(
|
||||
[
|
||||
'https://warehouse-theme-metal.myshopify.com/products/on-running-cloudmonster-2-mens',
|
||||
# This will likely fail
|
||||
'https://warehouse-theme-metal.myshopify.com/invalid-url',
|
||||
'https://warehouse-theme-metal.myshopify.com/products/valid-product',
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,37 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import HttpHeaders
|
||||
from crawlee.crawlers import BasicCrawlingContext, ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = ParselCrawler(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def setup_request(context: BasicCrawlingContext) -> None:
|
||||
# Add custom headers before making the request
|
||||
context.request.headers |= HttpHeaders(
|
||||
{
|
||||
'User-Agent': 'Crawlee Bot 1.0',
|
||||
'Accept': 'text/html,application/xhtml+xml',
|
||||
},
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
# Extract basic page information
|
||||
title = context.selector.css('title::text').get()
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
}
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,59 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import (
|
||||
PlaywrightCrawler,
|
||||
PlaywrightCrawlingContext,
|
||||
PlaywrightPreNavCrawlingContext,
|
||||
)
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
@crawler.pre_navigation_hook
|
||||
async def setup_page(context: PlaywrightPreNavCrawlingContext) -> None:
|
||||
# Set viewport size for consistent rendering
|
||||
await context.page.set_viewport_size({'width': 1280, 'height': 720})
|
||||
|
||||
# Block unnecessary resources to speed up crawling
|
||||
await context.block_requests(
|
||||
extra_url_patterns=[
|
||||
'*.png',
|
||||
'*.jpg',
|
||||
'*.jpeg',
|
||||
'*.gif',
|
||||
'*.svg',
|
||||
'*.css',
|
||||
'*.woff',
|
||||
'*.woff2',
|
||||
'*.ttf',
|
||||
'*google-analytics*',
|
||||
'*facebook*',
|
||||
'*twitter*',
|
||||
]
|
||||
)
|
||||
|
||||
# Set custom user agent
|
||||
await context.page.set_extra_http_headers(
|
||||
{
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; Crawlee Bot)',
|
||||
}
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
title = await context.page.title()
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
}
|
||||
)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,50 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.router import Router
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a custom router instance
|
||||
router = Router[ParselCrawlingContext]()
|
||||
|
||||
# Register a middleware that logs every request before it reaches a handler
|
||||
@router.use
|
||||
async def logging_middleware(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(
|
||||
f'Processing request: {context.request.url} label={context.request.label}'
|
||||
)
|
||||
|
||||
# Register a middleware that adds a timestamp to the request's user data
|
||||
@router.use
|
||||
async def timestamp_middleware(context: ParselCrawlingContext) -> None:
|
||||
context.request.user_data['start_time'] = time.monotonic()
|
||||
|
||||
@router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} with default handler')
|
||||
|
||||
@router.handler('CATEGORY')
|
||||
async def category_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} with category handler')
|
||||
|
||||
crawler = ParselCrawler(
|
||||
request_handler=router,
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
await crawler.run(
|
||||
[
|
||||
'https://warehouse-theme-metal.myshopify.com/',
|
||||
Request.from_url(
|
||||
'https://warehouse-theme-metal.myshopify.com/collections/all',
|
||||
label='CATEGORY',
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,36 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a crawler instance
|
||||
crawler = ParselCrawler(
|
||||
max_requests_per_crawl=10, # Limit the max requests per crawl.
|
||||
)
|
||||
|
||||
# Use the crawler's built-in router to define a default handler
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Extract page title
|
||||
title = context.selector.css('title::text').get() or 'No title found'
|
||||
|
||||
# Extract and save basic page data
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.url,
|
||||
'title': title,
|
||||
}
|
||||
)
|
||||
|
||||
# Find and enqueue product links for further crawling
|
||||
await context.enqueue_links(selector='a[href*="/products/"]', label='PRODUCT')
|
||||
|
||||
# Start crawling
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawler, BasicCrawlingContext
|
||||
from crawlee.request_loaders import ThrottlingRequestManager
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Open the default request queue.
|
||||
queue = await RequestQueue.open()
|
||||
|
||||
# Wrap it with ThrottlingRequestManager for specific domains. The throttler uses the
|
||||
# same storage backend as the underlying queue.
|
||||
throttler = ThrottlingRequestManager(
|
||||
queue,
|
||||
domains=['api.example.com', 'slow-site.org'],
|
||||
request_manager_opener=RequestQueue.open,
|
||||
)
|
||||
|
||||
# Pass the throttler as the crawler's request manager.
|
||||
crawler = BasicCrawler(request_manager=throttler)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: BasicCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Add requests. Listed domains are routed directly to their throttled sub-managers.
|
||||
# Others go to the inner manager.
|
||||
await throttler.add_requests(
|
||||
[
|
||||
'https://api.example.com/data',
|
||||
'https://api.example.com/users',
|
||||
'https://slow-site.org/page1',
|
||||
'https://fast-site.com/page1', # Not throttled
|
||||
]
|
||||
)
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,54 @@
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import TypedDict
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
|
||||
|
||||
class State(TypedDict):
|
||||
"""State available in the app."""
|
||||
|
||||
crawler: ParselCrawler
|
||||
requests_to_results: dict[str, asyncio.Future[dict[str, str]]]
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[State]:
|
||||
# Start up code that runs once when the app starts
|
||||
|
||||
# Results will be stored in this dictionary
|
||||
requests_to_results = dict[str, asyncio.Future[dict[str, str]]]()
|
||||
|
||||
crawler = ParselCrawler(
|
||||
# Keep the crawler alive even when there are no more requests to process now.
|
||||
# This makes the crawler wait for more requests to be added later.
|
||||
keep_alive=True
|
||||
)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: ParselCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
title = context.selector.xpath('//title/text()').get() or ''
|
||||
|
||||
# Extract data from the page and save it to the result dictionary.
|
||||
requests_to_results[context.request.unique_key].set_result(
|
||||
{
|
||||
'title': title,
|
||||
}
|
||||
)
|
||||
|
||||
# Start the crawler without awaiting it to finish
|
||||
crawler.log.info(f'Starting crawler for the {app.title}')
|
||||
run_task = asyncio.create_task(crawler.run([]))
|
||||
|
||||
# Make the crawler and the result dictionary available in the app state
|
||||
yield {'crawler': crawler, 'requests_to_results': requests_to_results}
|
||||
|
||||
# Cleanup code that runs once when the app shuts down
|
||||
crawler.stop()
|
||||
# Wait for the crawler to finish
|
||||
await run_task
|
||||
@@ -0,0 +1,58 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import HTMLResponse
|
||||
|
||||
import crawlee
|
||||
|
||||
from .crawler import lifespan
|
||||
|
||||
app = FastAPI(lifespan=lifespan, title='Crawler app')
|
||||
|
||||
|
||||
@app.get('/', response_class=HTMLResponse)
|
||||
def index() -> str:
|
||||
return """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>Scraper server</h1>
|
||||
<p>To scrape some page, visit "scrape" endpoint with url parameter.
|
||||
For example:
|
||||
<a href="/scrape?url=https://www.example.com">
|
||||
/scrape?url=https://www.example.com
|
||||
</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@app.get('/scrape')
|
||||
async def scrape_url(request: Request, url: str | None = None) -> dict:
|
||||
if not url:
|
||||
return {'url': 'missing', 'scrape result': 'no results'}
|
||||
|
||||
# Generate random unique key for the request
|
||||
unique_key = str(uuid4())
|
||||
|
||||
# Set the result future in the result dictionary so that it can be awaited
|
||||
request.state.requests_to_results[unique_key] = asyncio.Future[dict[str, str]]()
|
||||
|
||||
# Add the request to the crawler queue
|
||||
await request.state.crawler.add_requests(
|
||||
[crawlee.Request.from_url(url, unique_key=unique_key)]
|
||||
)
|
||||
|
||||
# Wait for the result future to be finished
|
||||
result = await request.state.requests_to_results[unique_key]
|
||||
|
||||
# Clean the result from the result dictionary to free up memory
|
||||
request.state.requests_to_results.pop(unique_key)
|
||||
|
||||
# Return the result
|
||||
return {'url': url, 'scrape result': result}
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import ConcurrencySettings
|
||||
from crawlee.crawlers import BeautifulSoupCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
concurrency_settings = ConcurrencySettings(
|
||||
# Set the maximum number of concurrent requests the crawler can run to 100.
|
||||
max_concurrency=100,
|
||||
# Limit the total number of requests to 10 per minute to avoid overwhelming
|
||||
# the target website.
|
||||
max_tasks_per_minute=10,
|
||||
)
|
||||
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Apply the defined concurrency settings to the crawler.
|
||||
concurrency_settings=concurrency_settings,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import ConcurrencySettings
|
||||
from crawlee.crawlers import BeautifulSoupCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
concurrency_settings = ConcurrencySettings(
|
||||
# Start with 8 concurrent tasks, as long as resources are available.
|
||||
desired_concurrency=8,
|
||||
# Maintain a minimum of 5 concurrent tasks to ensure steady crawling.
|
||||
min_concurrency=5,
|
||||
# Limit the maximum number of concurrent tasks to 10 to prevent
|
||||
# overloading the system.
|
||||
max_concurrency=10,
|
||||
)
|
||||
|
||||
crawler = BeautifulSoupCrawler(
|
||||
# Use the configured concurrency settings for the crawler.
|
||||
concurrency_settings=concurrency_settings,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Register the storage client via service locator.
|
||||
memory_storage_client = MemoryStorageClient()
|
||||
service_locator.set_storage_client(memory_storage_client)
|
||||
|
||||
# Retrieve the storage client.
|
||||
current_storage_client = service_locator.get_storage_client()
|
||||
|
||||
# Try to set a different storage client, which will raise ServiceConflictError
|
||||
# if storage client was already retrieved.
|
||||
file_system_storage_client = FileSystemStorageClient()
|
||||
service_locator.set_storage_client(file_system_storage_client)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee.configuration import Configuration
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
configuration = Configuration(
|
||||
log_level='DEBUG',
|
||||
headless=False,
|
||||
persist_state_interval=timedelta(seconds=30),
|
||||
)
|
||||
|
||||
# Register configuration via crawler.
|
||||
crawler = ParselCrawler(
|
||||
configuration=configuration,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.events import LocalEventManager
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
event_manager = LocalEventManager(
|
||||
system_info_interval=timedelta(seconds=5),
|
||||
)
|
||||
|
||||
# Register event manager via crawler.
|
||||
crawler = ParselCrawler(
|
||||
event_manager=event_manager,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,17 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
storage_client = MemoryStorageClient()
|
||||
|
||||
# Register storage client via crawler.
|
||||
crawler = ParselCrawler(
|
||||
storage_client=storage_client,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.configuration import Configuration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
configuration = Configuration(
|
||||
log_level='DEBUG',
|
||||
headless=False,
|
||||
persist_state_interval=timedelta(seconds=30),
|
||||
)
|
||||
|
||||
# Register configuration via service locator.
|
||||
service_locator.set_configuration(configuration)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,18 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.events import LocalEventManager
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
event_manager = LocalEventManager(
|
||||
system_info_interval=timedelta(seconds=5),
|
||||
)
|
||||
|
||||
# Register event manager via service locator.
|
||||
service_locator.set_event_manager(event_manager)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
storage_client = MemoryStorageClient()
|
||||
|
||||
# Register storage client via service locator.
|
||||
service_locator.set_storage_client(storage_client)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.configuration import Configuration
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
from crawlee.storages import Dataset
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
configuration = Configuration(
|
||||
log_level='DEBUG',
|
||||
headless=False,
|
||||
persist_state_interval=timedelta(seconds=30),
|
||||
)
|
||||
# Set the custom configuration as the global default configuration.
|
||||
service_locator.set_configuration(configuration)
|
||||
|
||||
# Use the global defaults when creating the dataset (or other storage).
|
||||
dataset_1 = await Dataset.open()
|
||||
|
||||
# Or set explicitly specific configuration if
|
||||
# you do not want to rely on global defaults.
|
||||
dataset_2 = await Dataset.open(
|
||||
storage_client=MemoryStorageClient(), configuration=configuration
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,17 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
from crawlee.storages import Dataset
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
storage_client = MemoryStorageClient()
|
||||
|
||||
# Pass the storage client to the dataset (or other storage) when opening it.
|
||||
dataset = await Dataset.open(
|
||||
storage_client=storage_client,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,85 @@
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from datetime import timedelta
|
||||
from itertools import count
|
||||
|
||||
from crawlee import ConcurrencySettings, Request
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import RequestCollisionError
|
||||
from crawlee.sessions import Session, SessionPool
|
||||
|
||||
|
||||
# Define a function for creating sessions with simple logic for unique `id` generation.
|
||||
# This is necessary if you need to specify a particular session for the first request,
|
||||
# for example during authentication
|
||||
def create_session_function() -> Callable[[], Session]:
|
||||
counter = count()
|
||||
|
||||
def create_session() -> Session:
|
||||
return Session(
|
||||
id=str(next(counter)),
|
||||
max_usage_count=999_999,
|
||||
max_age=timedelta(hours=999_999),
|
||||
max_error_score=100,
|
||||
blocked_status_codes=[403],
|
||||
)
|
||||
|
||||
return create_session
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
# Adjust request limits according to your pool size
|
||||
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=500),
|
||||
# Requests are bound to specific sessions, no rotation needed
|
||||
max_session_rotations=0,
|
||||
session_pool=SessionPool(
|
||||
max_pool_size=10, create_session_function=create_session_function()
|
||||
),
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def basic_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Initialize the session and bind the next request to this session if needed
|
||||
@crawler.router.handler(label='session_init')
|
||||
async def session_init(context: HttpCrawlingContext) -> None:
|
||||
next_requests = []
|
||||
if context.session:
|
||||
context.log.info(f'Init session {context.session.id}')
|
||||
next_request = Request.from_url(
|
||||
'https://a.placeholder.com', session_id=context.session.id
|
||||
)
|
||||
next_requests.append(next_request)
|
||||
|
||||
await context.add_requests(next_requests)
|
||||
|
||||
# Handle errors when a session is blocked and no longer available in the pool
|
||||
# when attempting to execute requests bound to it
|
||||
@crawler.failed_request_handler
|
||||
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
if isinstance(error, RequestCollisionError) and context.session:
|
||||
context.log.error(
|
||||
f'Request {context.request.url} failed, because the bound '
|
||||
'session is unavailable'
|
||||
)
|
||||
|
||||
# Create a pool of requests bound to their respective sessions
|
||||
# Use `always_enqueue=True` if session initialization happens on a non-unique address,
|
||||
# such as the site's main page
|
||||
init_requests = [
|
||||
Request.from_url(
|
||||
'https://example.org/',
|
||||
label='session_init',
|
||||
session_id=str(session_id),
|
||||
use_extended_unique_key=True,
|
||||
)
|
||||
for session_id in range(1, 11)
|
||||
]
|
||||
|
||||
await crawler.run(init_requests)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,56 @@
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from crawlee import ConcurrencySettings, Request
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import SessionError
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
# Limit requests per minute to reduce the chance of being blocked
|
||||
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=50),
|
||||
# Disable session rotation
|
||||
max_session_rotations=0,
|
||||
session_pool=SessionPool(
|
||||
# Only one session in the pool
|
||||
max_pool_size=1,
|
||||
create_session_settings={
|
||||
# High value for session usage limit
|
||||
'max_usage_count': 999_999,
|
||||
# High value for session lifetime
|
||||
'max_age': timedelta(hours=999_999),
|
||||
# High score allows the session to encounter more errors
|
||||
# before crawlee decides the session is blocked
|
||||
# Make sure you know how to handle these errors
|
||||
'max_error_score': 100,
|
||||
# 403 status usually indicates you're already blocked
|
||||
'blocked_status_codes': [403],
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
# Basic request handling logic
|
||||
@crawler.router.default_handler
|
||||
async def basic_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Handler for session initialization (authentication, initial cookies, etc.)
|
||||
@crawler.router.handler(label='session_init')
|
||||
async def session_init(context: HttpCrawlingContext) -> None:
|
||||
if context.session:
|
||||
context.log.info(f'Init session {context.session.id}')
|
||||
|
||||
# Monitor if our session gets blocked and explicitly stop the crawler
|
||||
@crawler.error_handler
|
||||
async def error_processing(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
if isinstance(error, SessionError) and context.session:
|
||||
context.log.info(f'Session {context.session.id} blocked')
|
||||
crawler.stop()
|
||||
|
||||
await crawler.run([Request.from_url('https://example.org/', label='session_init')])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.crawlers import BasicCrawler, BasicCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
# options
|
||||
)
|
||||
|
||||
# Initialize crawler with a custom SessionPool configuration
|
||||
# to manage concurrent sessions and proxy rotation
|
||||
crawler = BasicCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
# Activates the Session pool (default is true).
|
||||
use_session_pool=True,
|
||||
# Overrides default Session pool configuration.
|
||||
session_pool=SessionPool(max_pool_size=100),
|
||||
)
|
||||
|
||||
# Define the default request handler that manages session states
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BasicCrawlingContext) -> None:
|
||||
# Send request, BasicCrawler automatically selects a session from the pool
|
||||
# and sets a proxy for it. You can check it with `context.session`
|
||||
# and `context.proxy_info`.
|
||||
response = await context.send_request(context.request.url)
|
||||
|
||||
page_content = (await response.read()).decode()
|
||||
title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
|
||||
|
||||
if context.session and (title := title_match.group(1) if title_match else None):
|
||||
if title == 'Blocked':
|
||||
context.session.retire()
|
||||
elif title == 'Not sure if blocked, might also be a connection error':
|
||||
context.session.mark_bad()
|
||||
else:
|
||||
context.session.mark_good() # BasicCrawler handles this automatically.
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
# options
|
||||
)
|
||||
|
||||
# Initialize crawler with a custom SessionPool configuration
|
||||
# to manage concurrent sessions and proxy rotation
|
||||
crawler = BeautifulSoupCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
# Activates the Session pool (default is true).
|
||||
use_session_pool=True,
|
||||
# Overrides default Session pool configuration.
|
||||
session_pool=SessionPool(max_pool_size=100),
|
||||
)
|
||||
|
||||
# Define the default request handler that manages session states
|
||||
# based on the response content and potential blocking
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
title = context.soup.title.get_text() if context.soup.title else None
|
||||
|
||||
if context.session:
|
||||
if title == 'Blocked':
|
||||
context.session.retire()
|
||||
elif title == 'Not sure if blocked, might also be a connection error':
|
||||
context.session.mark_bad()
|
||||
else:
|
||||
context.session.mark_good() # BasicCrawler handles this automatically.
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,44 @@
|
||||
import asyncio
|
||||
import re
|
||||
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
# options
|
||||
)
|
||||
|
||||
# Initialize crawler with a custom SessionPool configuration
|
||||
# to manage concurrent sessions and proxy rotation
|
||||
crawler = HttpCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
# Activates the Session pool (default is true).
|
||||
use_session_pool=True,
|
||||
# Overrides default Session pool configuration.
|
||||
session_pool=SessionPool(max_pool_size=100),
|
||||
)
|
||||
|
||||
# Define the default request handler that manages session states
|
||||
# based on the response content and potential blocking
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
page_content = (await context.http_response.read()).decode()
|
||||
title_match = re.search(r'<title(?:.*?)>(.*?)</title>', page_content)
|
||||
|
||||
if context.session and (title := title_match.group(1) if title_match else None):
|
||||
if title == 'Blocked':
|
||||
context.session.retire()
|
||||
elif title == 'Not sure if blocked, might also be a connection error':
|
||||
context.session.mark_bad()
|
||||
else:
|
||||
context.session.mark_good() # BasicCrawler handles this automatically.
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
# options
|
||||
)
|
||||
|
||||
# Initialize crawler with a custom SessionPool configuration
|
||||
# to manage concurrent sessions and proxy rotation
|
||||
crawler = ParselCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
# Activates the Session pool (default is true).
|
||||
use_session_pool=True,
|
||||
# Overrides default Session pool configuration.
|
||||
session_pool=SessionPool(max_pool_size=100),
|
||||
)
|
||||
|
||||
# Define the default request handler that manages session states
|
||||
# based on the response content and potential blocking
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: ParselCrawlingContext) -> None:
|
||||
title = context.selector.css('title::text').get()
|
||||
|
||||
if context.session:
|
||||
if title == 'Blocked':
|
||||
context.session.retire()
|
||||
elif title == 'Not sure if blocked, might also be a connection error':
|
||||
context.session.mark_bad()
|
||||
else:
|
||||
context.session.mark_good() # BasicCrawler handles this automatically.
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,42 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# To use the proxy IP session rotation logic, you must turn the proxy usage on.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
# options
|
||||
)
|
||||
|
||||
# Initialize crawler with a custom SessionPool configuration
|
||||
# to manage concurrent sessions and proxy rotation
|
||||
crawler = PlaywrightCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
# Activates the Session pool (default is true).
|
||||
use_session_pool=True,
|
||||
# Overrides default Session pool configuration.
|
||||
session_pool=SessionPool(max_pool_size=100),
|
||||
)
|
||||
|
||||
# Define the default request handler that manages session states
|
||||
# based on the response content and potential blocking
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
title = await context.page.title()
|
||||
|
||||
if context.session:
|
||||
if title == 'Blocked':
|
||||
context.session.retire()
|
||||
elif title == 'Not sure if blocked, might also be a connection error':
|
||||
context.session.mark_bad()
|
||||
else:
|
||||
context.session.mark_good() # BasicCrawler handles this automatically.
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Override the default Session pool configuration.
|
||||
async with SessionPool(
|
||||
max_pool_size=100,
|
||||
create_session_settings={'max_usage_count': 10, 'blocked_status_codes': [403]},
|
||||
) as session_pool:
|
||||
session = await session_pool.get_session()
|
||||
|
||||
# Increase the error_score.
|
||||
session.mark_bad()
|
||||
|
||||
# Throw away the session.
|
||||
session.retire()
|
||||
|
||||
# Lower the error_score and mark the session good.
|
||||
session.mark_good()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
import asyncio
|
||||
from typing import cast
|
||||
|
||||
from crawlee.browsers import StagehandOptions
|
||||
from crawlee.crawlers import StagehandCrawler, StagehandCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = StagehandCrawler(
|
||||
stagehand_options=StagehandOptions(
|
||||
model_api_key='your-openai-api-key',
|
||||
model='openai/gpt-5.4-nano',
|
||||
),
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: StagehandCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
# Dismiss overlays or interact with the page using natural language.
|
||||
await context.page.act(input='Click the accept cookies button if present')
|
||||
|
||||
# Extract data from the page using AI.
|
||||
extracted = await context.page.extract(
|
||||
instruction='Get the page title and the main heading text',
|
||||
schema={
|
||||
'type': 'object',
|
||||
'properties': {
|
||||
'title': {'type': 'string'},
|
||||
'heading': {'type': 'string'},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
extract_result = extracted.data.result
|
||||
|
||||
if isinstance(extract_result, dict):
|
||||
# Push extracted data to the dataset
|
||||
# Use `cast()` to provide a more specific type hint for the extracted data.
|
||||
await context.push_data(cast('dict[str, str | None]', extract_result))
|
||||
|
||||
await crawler.run(['https://example.com'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,37 @@
|
||||
import asyncio
|
||||
from typing import cast
|
||||
|
||||
from crawlee.browsers import StagehandOptions
|
||||
from crawlee.crawlers import StagehandCrawler, StagehandCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Use Browserbase cloud browser instead of a local Chromium instance.
|
||||
crawler = StagehandCrawler(
|
||||
stagehand_options=StagehandOptions(
|
||||
env='BROWSERBASE',
|
||||
browserbase_api_key='your-browserbase-api-key',
|
||||
project_id='your-project-id',
|
||||
model_api_key='your-openai-api-key',
|
||||
model='openai/gpt-5.4-nano',
|
||||
),
|
||||
max_requests_per_crawl=5,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def handler(context: StagehandCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
extracted = await context.page.extract(
|
||||
instruction='Get the main content of the page',
|
||||
)
|
||||
|
||||
extract_result = extracted.data.result
|
||||
|
||||
await context.push_data(cast('dict[str, str | None]', extract_result))
|
||||
|
||||
await crawler.run(['https://example.com'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from crawlee.storage_clients import StorageClient
|
||||
from crawlee.storage_clients._base import (
|
||||
DatasetClient,
|
||||
KeyValueStoreClient,
|
||||
RequestQueueClient,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from crawlee.configuration import Configuration
|
||||
|
||||
# Implement the storage type clients with your backend logic.
|
||||
|
||||
|
||||
class CustomDatasetClient(DatasetClient):
|
||||
# Implement methods like push_data, get_data, iterate_items, etc.
|
||||
pass
|
||||
|
||||
|
||||
class CustomKeyValueStoreClient(KeyValueStoreClient):
|
||||
# Implement methods like get_value, set_value, delete, etc.
|
||||
pass
|
||||
|
||||
|
||||
class CustomRequestQueueClient(RequestQueueClient):
|
||||
# Implement methods like add_request, fetch_next_request, etc.
|
||||
pass
|
||||
|
||||
|
||||
# Implement the storage client factory.
|
||||
|
||||
|
||||
class CustomStorageClient(StorageClient):
|
||||
async def create_dataset_client(
|
||||
self,
|
||||
*,
|
||||
id: str | None = None,
|
||||
name: str | None = None,
|
||||
configuration: Configuration | None = None,
|
||||
) -> CustomDatasetClient:
|
||||
# Create and return your custom dataset client.
|
||||
pass
|
||||
|
||||
async def create_kvs_client(
|
||||
self,
|
||||
*,
|
||||
id: str | None = None,
|
||||
name: str | None = None,
|
||||
configuration: Configuration | None = None,
|
||||
) -> CustomKeyValueStoreClient:
|
||||
# Create and return your custom key-value store client.
|
||||
pass
|
||||
|
||||
async def create_rq_client(
|
||||
self,
|
||||
*,
|
||||
id: str | None = None,
|
||||
name: str | None = None,
|
||||
configuration: Configuration | None = None,
|
||||
) -> CustomRequestQueueClient:
|
||||
# Create and return your custom request queue client.
|
||||
pass
|
||||
@@ -0,0 +1,8 @@
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import FileSystemStorageClient
|
||||
|
||||
# Create a new instance of storage client.
|
||||
storage_client = FileSystemStorageClient()
|
||||
|
||||
# And pass it to the crawler.
|
||||
crawler = ParselCrawler(storage_client=storage_client)
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
from crawlee.configuration import Configuration
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import FileSystemStorageClient
|
||||
|
||||
# Create a new instance of storage client.
|
||||
storage_client = FileSystemStorageClient()
|
||||
|
||||
# Create a configuration with custom settings.
|
||||
configuration = Configuration(
|
||||
storage_dir='./my_storage',
|
||||
purge_on_start=False,
|
||||
)
|
||||
|
||||
# And pass them to the crawler.
|
||||
crawler = ParselCrawler(
|
||||
storage_client=storage_client,
|
||||
configuration=configuration,
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
# Create a new instance of storage client.
|
||||
storage_client = MemoryStorageClient()
|
||||
|
||||
# And pass it to the crawler.
|
||||
crawler = ParselCrawler(storage_client=storage_client)
|
||||
@@ -0,0 +1,10 @@
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import RedisStorageClient
|
||||
|
||||
# Create a new instance of storage client using connection string.
|
||||
# 'redis://localhost:6379' is the just placeholder, replace it with your actual
|
||||
# connection string.
|
||||
storage_client = RedisStorageClient(connection_string='redis://localhost:6379')
|
||||
|
||||
# And pass it to the crawler.
|
||||
crawler = ParselCrawler(storage_client=storage_client)
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from crawlee.configuration import Configuration
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import RedisStorageClient
|
||||
|
||||
# Create a new instance of storage client using a Redis client with custom settings.
|
||||
# Replace host and port with your actual Redis server configuration.
|
||||
# Other Redis client settings can be adjusted as needed.
|
||||
storage_client = RedisStorageClient(
|
||||
redis=Redis(
|
||||
host='localhost',
|
||||
port=6379,
|
||||
retry_on_timeout=True,
|
||||
socket_keepalive=True,
|
||||
socket_connect_timeout=10,
|
||||
)
|
||||
)
|
||||
|
||||
# Create a configuration with custom settings.
|
||||
configuration = Configuration(purge_on_start=False)
|
||||
|
||||
# And pass them to the crawler.
|
||||
crawler = ParselCrawler(
|
||||
storage_client=storage_client,
|
||||
configuration=configuration,
|
||||
)
|
||||
@@ -0,0 +1,29 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import service_locator
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
from crawlee.storages import Dataset
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create custom storage client, MemoryStorageClient for example.
|
||||
storage_client = MemoryStorageClient()
|
||||
|
||||
# Register it globally via the service locator.
|
||||
service_locator.set_storage_client(storage_client)
|
||||
|
||||
# Or pass it directly to the crawler, it will be registered globally
|
||||
# to the service locator under the hood.
|
||||
crawler = ParselCrawler(storage_client=storage_client)
|
||||
|
||||
# Or just provide it when opening a storage (e.g. dataset), it will be used
|
||||
# for this storage only, not globally.
|
||||
dataset = await Dataset.open(
|
||||
name='my-dataset',
|
||||
storage_client=storage_client,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,12 @@
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import SqlStorageClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a new instance of storage client.
|
||||
# This will create an SQLite database file crawlee.db or created tables in your
|
||||
# database if you pass `connection_string` or `engine`
|
||||
# Use the context manager to ensure that connections are properly cleaned up.
|
||||
async with SqlStorageClient() as storage_client:
|
||||
# And pass it to the crawler.
|
||||
crawler = ParselCrawler(storage_client=storage_client)
|
||||
@@ -0,0 +1,33 @@
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from crawlee.configuration import Configuration
|
||||
from crawlee.crawlers import ParselCrawler
|
||||
from crawlee.storage_clients import SqlStorageClient
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a new instance of storage client.
|
||||
# On first run, also creates tables in your PostgreSQL database.
|
||||
# Use the context manager to ensure that connections are properly cleaned up.
|
||||
async with SqlStorageClient(
|
||||
# Create an `engine` with the desired configuration
|
||||
engine=create_async_engine(
|
||||
'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres',
|
||||
future=True,
|
||||
pool_size=5,
|
||||
max_overflow=10,
|
||||
pool_recycle=3600,
|
||||
pool_pre_ping=True,
|
||||
echo=False,
|
||||
)
|
||||
) as storage_client:
|
||||
# Create a configuration with custom settings.
|
||||
configuration = Configuration(
|
||||
purge_on_start=False,
|
||||
)
|
||||
|
||||
# And pass them to the crawler.
|
||||
crawler = ParselCrawler(
|
||||
storage_client=storage_client,
|
||||
configuration=configuration,
|
||||
)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user