86 lines
3.2 KiB
Python
86 lines
3.2 KiB
Python
import asyncio
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
|
|
from crawlee import ConcurrencySettings, Request
|
|
from crawlee.crawlers import (
|
|
HttpCrawler,
|
|
HttpCrawlingContext,
|
|
)
|
|
from crawlee.sessions import SessionPool
|
|
|
|
|
|
async def main() -> None:
|
|
crawler = HttpCrawler(
|
|
max_requests_per_crawl=10,
|
|
# Configure to use a single persistent session throughout the crawl
|
|
max_session_rotations=0,
|
|
# Limit request rate to avoid triggering anti-scraping measures
|
|
concurrency_settings=ConcurrencySettings(max_tasks_per_minute=30),
|
|
session_pool=SessionPool(
|
|
max_pool_size=1,
|
|
create_session_settings={
|
|
# Set high value to ensure the session isn't replaced during crawling
|
|
'max_usage_count': 999_999,
|
|
# Set high value to prevent session expiration during crawling
|
|
'max_age': timedelta(hours=999_999),
|
|
# Higher error tolerance before the session is considered blocked
|
|
# Make sure you implement proper error handling in your code
|
|
'max_error_score': 100,
|
|
},
|
|
),
|
|
)
|
|
|
|
# Default request handler for normal page processing
|
|
@crawler.router.default_handler
|
|
async def request_handler(context: HttpCrawlingContext) -> None:
|
|
context.log.info(f'Processing {context.request.url} ...')
|
|
|
|
# Specialized handler for the login API request
|
|
@crawler.router.handler('login')
|
|
async def login_handler(context: HttpCrawlingContext) -> None:
|
|
context.log.info(f'Processing login at {context.request.url} ...')
|
|
|
|
# Verify that a session is available before proceeding
|
|
if not context.session:
|
|
raise RuntimeError('Session not found')
|
|
|
|
# Parse the API response containing authentication tokens and user data
|
|
data = json.loads(await context.http_response.read())
|
|
|
|
# Extract authentication data from the response
|
|
token = data['token']
|
|
expires = data['expires'].replace('Z', '+00:00')
|
|
expires_int = int(datetime.fromisoformat(expires).timestamp())
|
|
user_id = data['userId']
|
|
username = data['username']
|
|
|
|
# Set authentication cookies in the session that will be used
|
|
# for subsequent requests
|
|
context.session.cookies.set(name='token', value=token, expires=expires_int)
|
|
context.session.cookies.set(name='userID', value=user_id)
|
|
context.session.cookies.set(name='userName', value=username)
|
|
|
|
# After successful authentication, continue crawling with the
|
|
# authenticated session
|
|
await context.add_requests(['https://demoqa.com/BookStore/v1/Books'])
|
|
|
|
# Create a POST request to the authentication API endpoint
|
|
# This will trigger the login_handler when executed
|
|
request = Request.from_url(
|
|
'https://demoqa.com/Account/v1/Login',
|
|
label='login',
|
|
method='POST',
|
|
payload=json.dumps(
|
|
{'userName': 'crawlee_test', 'password': 'Test1234!'}
|
|
).encode(),
|
|
headers={'Content-Type': 'application/json'},
|
|
)
|
|
|
|
# Start the crawling process with the login request
|
|
await crawler.run([request])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(main())
|