同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from crawlee import HttpHeaders
|
||||
from crawlee.crawlers import HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import HttpStatusCodeError
|
||||
from crawlee.sessions import SessionPool
|
||||
|
||||
# Using a placeholder refresh token for this example
|
||||
REFRESH_TOKEN = 'PLACEHOLDER'
|
||||
UNAUTHORIZED_CODE = 401
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(
|
||||
max_request_retries=2,
|
||||
# Only treat 403 as a blocking status code, not 401
|
||||
session_pool=SessionPool(create_session_settings={'blocked_status_codes': [403]}),
|
||||
# Don't treat 401 responses as errors
|
||||
ignore_http_error_status_codes=[UNAUTHORIZED_CODE],
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
# Now we can handle 401 responses ourselves
|
||||
if context.http_response.status_code == UNAUTHORIZED_CODE:
|
||||
# Get a fresh access token
|
||||
headers = {'authorization': f'Bearer {REFRESH_TOKEN}'}
|
||||
response = await context.send_request(
|
||||
'https://placeholder.org/refresh', headers=headers
|
||||
)
|
||||
data = json.loads(await response.read())
|
||||
# Add the new token to our `Request` headers
|
||||
context.request.headers |= HttpHeaders(
|
||||
{'authorization': f'Bearer {data["access_token"]}'},
|
||||
)
|
||||
# Trigger a retry with our updated headers
|
||||
raise HttpStatusCodeError('Unauthorized', status_code=UNAUTHORIZED_CODE)
|
||||
|
||||
await crawler.run(['http://httpbingo.org/status/401'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import HttpStatusCodeError, SessionError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = HttpCrawler(max_request_retries=5)
|
||||
|
||||
# Create a parsing error for demonstration
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
raise ValueError('Simulated parsing error')
|
||||
|
||||
# This handler runs before any retry attempts
|
||||
@crawler.error_handler
|
||||
async def retry_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
context.log.error(f'Failed request {context.request.url}')
|
||||
# Only allow retries for network-related errors
|
||||
if not isinstance(error, (SessionError, HttpStatusCodeError)):
|
||||
context.log.error('Non-network error detected')
|
||||
# Stop further retry attempts for this `Request`
|
||||
context.request.no_retry = True
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,40 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import Request
|
||||
from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext
|
||||
from crawlee.errors import ProxyError
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Set how many session rotations will happen before calling the error handler
|
||||
# when ProxyError occurs
|
||||
crawler = HttpCrawler(max_session_rotations=5, max_request_retries=6)
|
||||
|
||||
# For this example, we'll create a proxy error in our handler
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: HttpCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
raise ProxyError('Simulated proxy error')
|
||||
|
||||
# This handler runs after all retry attempts are exhausted
|
||||
@crawler.failed_request_handler
|
||||
async def failed_handler(context: BasicCrawlingContext, error: Exception) -> None:
|
||||
context.log.error(f'Failed request {context.request.url}, after 5 rotations')
|
||||
request = context.request
|
||||
# For proxy errors, we can add a new `Request` to try again
|
||||
if isinstance(error, ProxyError) and not request.unique_key.startswith('retry'):
|
||||
context.log.info(f'Retrying {request.url} ...')
|
||||
# Create a new `Request` with a modified key to avoid deduplication
|
||||
new_request = Request.from_url(
|
||||
request.url, unique_key=f'retry{request.unique_key}'
|
||||
)
|
||||
|
||||
# Add the new `Request` to the `Queue`
|
||||
rq = await crawler.get_request_manager()
|
||||
await rq.add_request(new_request)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user