73 lines
2.3 KiB
Python
73 lines
2.3 KiB
Python
from crawlee.crawlers import PlaywrightCrawlingContext
|
|
from crawlee.router import Router
|
|
|
|
router = Router[PlaywrightCrawlingContext]()
|
|
|
|
|
|
@router.default_handler
|
|
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
|
# This is a fallback route which will handle the start URL.
|
|
context.log.info(f'default_handler is processing {context.request.url}')
|
|
|
|
await context.page.wait_for_selector('.collection-block-item')
|
|
|
|
await context.enqueue_links(
|
|
selector='.collection-block-item',
|
|
label='CATEGORY',
|
|
)
|
|
|
|
|
|
@router.handler('CATEGORY')
|
|
async def category_handler(context: PlaywrightCrawlingContext) -> None:
|
|
# This replaces the context.request.label == CATEGORY branch of the if clause.
|
|
context.log.info(f'category_handler is processing {context.request.url}')
|
|
|
|
await context.page.wait_for_selector('.product-item > a')
|
|
|
|
await context.enqueue_links(
|
|
selector='.product-item > a',
|
|
label='DETAIL',
|
|
)
|
|
|
|
next_button = await context.page.query_selector('a.pagination__next')
|
|
|
|
if next_button:
|
|
await context.enqueue_links(
|
|
selector='a.pagination__next',
|
|
label='CATEGORY',
|
|
)
|
|
|
|
|
|
@router.handler('DETAIL')
|
|
async def detail_handler(context: PlaywrightCrawlingContext) -> None:
|
|
# This replaces the context.request.label == DETAIL branch of the if clause.
|
|
context.log.info(f'detail_handler is processing {context.request.url}')
|
|
|
|
url_part = context.request.url.split('/').pop()
|
|
manufacturer = url_part.split('-')[0]
|
|
|
|
title = await context.page.locator('.product-meta h1').text_content()
|
|
|
|
sku = await context.page.locator('span.product-meta__sku-number').text_content()
|
|
|
|
price_element = context.page.locator('span.price', has_text='$').first
|
|
current_price_string = await price_element.text_content() or ''
|
|
raw_price = current_price_string.split('$')[1]
|
|
price = float(raw_price.replace(',', ''))
|
|
|
|
in_stock_element = context.page.locator(
|
|
selector='span.product-form__inventory',
|
|
has_text='In stock',
|
|
).first
|
|
in_stock = await in_stock_element.count() > 0
|
|
|
|
data = {
|
|
'manufacturer': manufacturer,
|
|
'title': title,
|
|
'sku': sku,
|
|
'price': price,
|
|
'in_stock': in_stock,
|
|
}
|
|
|
|
await context.push_data(data)
|