Files
crawlee-python/docs/guides/code_examples/login_crawler/http_login.py
T

86 lines
3.2 KiB
Python

import asyncio
import json
from datetime import datetime, timedelta
from crawlee import ConcurrencySettings, Request
from crawlee.crawlers import (
HttpCrawler,
HttpCrawlingContext,
)
from crawlee.sessions import SessionPool
async def main() -> None:
crawler = HttpCrawler(
max_requests_per_crawl=10,
# Configure to use a single persistent session throughout the crawl
max_session_rotations=0,
# Limit request rate to avoid triggering anti-scraping measures
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
session_pool=SessionPool(
max_pool_size=1,
create_session_settings={
# Set high value to ensure the session isn't replaced during crawling
'max_usage_count': 999_999,
# Set high value to prevent session expiration during crawling
'max_age': timedelta(hours=999_999),
# Higher error tolerance before the session is considered blocked
# Make sure you implement proper error handling in your code
'max_error_score': 100,
},
),
)
# Default request handler for normal page processing
@crawler.router.default_handler
async def request_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Specialized handler for the login API request
@crawler.router.handler('login')
async def login_handler(context: HttpCrawlingContext) -> None:
context.log.info(f'Processing login at {context.request.url} ...')
# Verify that a session is available before proceeding
if not context.session:
raise RuntimeError('Session not found')
# Parse the API response containing authentication tokens and user data
data = json.loads(await context.http_response.read())
# Extract authentication data from the response
token = data['token']
expires = data['expires'].replace('Z', '+00:00')
expires_int = int(datetime.fromisoformat(expires).timestamp())
user_id = data['userId']
username = data['username']
# Set authentication cookies in the session that will be used
# for subsequent requests
context.session.cookies.set(name='token', value=token, expires=expires_int)
context.session.cookies.set(name='userID', value=user_id)
context.session.cookies.set(name='userName', value=username)
# After successful authentication, continue crawling with the
# authenticated session
await context.add_requests(['https://demoqa.com/BookStore/v1/Books'])
# Create a POST request to the authentication API endpoint
# This will trigger the login_handler when executed
request = Request.from_url(
'https://demoqa.com/Account/v1/Login',
label='login',
method='POST',
payload=json.dumps(
{'userName': 'crawlee_test', 'password': 'Test1234!'}
).encode(),
headers={'Content-Type': 'application/json'},
)
# Start the crawling process with the login request
await crawler.run([request])
if __name__ == '__main__':
asyncio.run(main())