同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,30 @@
|
||||
import asyncio
|
||||
|
||||
# Add import of crawler and crawling context.
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# First you create the request queue instance.
|
||||
rq = await RequestQueue.open()
|
||||
|
||||
# And then you add one or more requests to it.
|
||||
await rq.add_request('https://crawlee.dev')
|
||||
|
||||
crawler = BeautifulSoupCrawler(request_manager=rq)
|
||||
|
||||
# Define a request handler and attach it to the crawler using the decorator.
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Extract <title> text with BeautifulSoup.
|
||||
# See BeautifulSoup documentation for API docs.
|
||||
url = context.request.url
|
||||
title = context.soup.title.string if context.soup.title else ''
|
||||
context.log.info(f'The title of {url} is: {title}.')
|
||||
|
||||
await crawler.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
# You don't need to import RequestQueue anymore.
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
url = context.request.url
|
||||
title = context.soup.title.string if context.soup.title else ''
|
||||
context.log.info(f'The title of {url} is: {title}.')
|
||||
|
||||
# Start the crawler with the provided URLs.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.storages import RequestQueue
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# First you create the request queue instance.
|
||||
rq = await RequestQueue.open()
|
||||
|
||||
# And then you add one or more requests to it.
|
||||
await rq.add_request('https://crawlee.dev')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}.')
|
||||
|
||||
# See the `EnqueueStrategy` type alias for more strategy options.
|
||||
# highlight-next-line
|
||||
await context.enqueue_links(
|
||||
# highlight-next-line
|
||||
strategy='same-domain',
|
||||
# highlight-next-line
|
||||
)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Let's limit our crawls to make our tests shorter and safer.
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
url = context.request.url
|
||||
title = context.soup.title.string if context.soup.title else ''
|
||||
context.log.info(f'The title of {url} is: {title}.')
|
||||
|
||||
# The enqueue_links function is available as one of the fields of the context.
|
||||
# It is also context aware, so it does not require any parameters.
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,29 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee import Glob
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}.')
|
||||
|
||||
# Enqueue links that match the 'include' glob pattern and
|
||||
# do not match the 'exclude' glob pattern.
|
||||
# highlight-next-line
|
||||
await context.enqueue_links(
|
||||
# highlight-next-line
|
||||
include=[Glob('https://someplace.com/**/cats')],
|
||||
# highlight-next-line
|
||||
exclude=[Glob('https://**/archive/**')],
|
||||
# highlight-next-line
|
||||
)
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,19 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
url = context.request.url
|
||||
title = context.soup.title.string if context.soup.title else ''
|
||||
context.log.info(f'The title of {url} is: {title}.')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from crawlee import HttpHeaders, RequestOptions, RequestTransformAction
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
def transform_request(
|
||||
request_options: RequestOptions,
|
||||
) -> RequestOptions | RequestTransformAction:
|
||||
# Skip requests to PDF files
|
||||
if request_options['url'].endswith('.pdf'):
|
||||
return 'skip'
|
||||
|
||||
if '/docs' in request_options['url']:
|
||||
# Add custom headers to requests to specific URLs
|
||||
request_options['headers'] = HttpHeaders({'Custom-Header': 'value'})
|
||||
|
||||
elif '/blog' in request_options['url']:
|
||||
# Add label for certain URLs
|
||||
request_options['label'] = 'BLOG'
|
||||
|
||||
else:
|
||||
# Signal that the request should proceed without any transformation
|
||||
return 'unchanged'
|
||||
|
||||
return request_options
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}.')
|
||||
|
||||
# Transform request before enqueueing
|
||||
await context.enqueue_links(transform_request_function=transform_request)
|
||||
|
||||
@crawler.router.handler('BLOG')
|
||||
async def blog_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Blog Processing {context.request.url}.')
|
||||
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
|
||||
# Instead of BeautifulSoupCrawler let's use Playwright to be able to render JavaScript.
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Wait for the collection cards to render on the page. This ensures that
|
||||
# the elements we want to interact with are present in the DOM.
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
# Execute a function within the browser context to target the collection
|
||||
# card elements and extract their text content, trimming any leading or
|
||||
# trailing whitespace.
|
||||
category_texts = await context.page.eval_on_selector_all(
|
||||
'.collection-block-item',
|
||||
'(els) => els.map(el => el.textContent.trim())',
|
||||
)
|
||||
|
||||
# Log the extracted texts.
|
||||
for i, text in enumerate(category_texts):
|
||||
context.log.info(f'CATEGORY_{i + 1}: {text}')
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,57 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# We're not processing detail pages yet, so we just pass.
|
||||
if context.request.label == 'DETAIL':
|
||||
pass
|
||||
|
||||
# We are now on a category page. We can use this to paginate through and
|
||||
# enqueue all products, as well as any subsequent pages we find.
|
||||
elif context.request.label == 'CATEGORY':
|
||||
# Wait for the product items to render.
|
||||
await context.page.wait_for_selector('.product-item > a')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label DETAIL.
|
||||
await context.enqueue_links(
|
||||
selector='.product-item > a',
|
||||
label='DETAIL',
|
||||
)
|
||||
|
||||
# Find the "Next" button to paginate through the category pages.
|
||||
next_button = await context.page.query_selector('a.pagination__next')
|
||||
|
||||
# If a "Next" button is found, enqueue the next page of results.
|
||||
if next_button:
|
||||
await context.enqueue_links(
|
||||
selector='a.pagination__next',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
# This indicates we're on the start page with no specific label.
|
||||
# On the start page, we want to enqueue all the category pages.
|
||||
else:
|
||||
# Wait for the collection cards to render.
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label CATEGORY.
|
||||
await context.enqueue_links(
|
||||
selector='.collection-block-item',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# Wait for the category cards to render on the page. This ensures that
|
||||
# the elements we want to interact with are present in the DOM.
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
# Enqueue links found within elements that match the specified selector.
|
||||
# These links will be added to the crawling queue with the label CATEGORY.
|
||||
await context.enqueue_links(
|
||||
selector='.collection-block-item',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,97 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Let's limit our crawls to make our tests shorter and safer.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# We're not processing detail pages yet, so we just pass.
|
||||
if context.request.label == 'DETAIL':
|
||||
# Split the URL and get the last part to extract the manufacturer.
|
||||
url_part = context.request.url.split('/').pop()
|
||||
manufacturer = url_part.split('-')[0]
|
||||
|
||||
# Extract the title using the combined selector.
|
||||
title = await context.page.locator('.product-meta h1').text_content()
|
||||
|
||||
# Extract the SKU using its selector.
|
||||
sku = await context.page.locator(
|
||||
'span.product-meta__sku-number'
|
||||
).text_content()
|
||||
|
||||
# Locate the price element that contains the '$' sign and filter out
|
||||
# the visually hidden elements.
|
||||
price_element = context.page.locator('span.price', has_text='$').first
|
||||
current_price_string = await price_element.text_content() or ''
|
||||
raw_price = current_price_string.split('$')[1]
|
||||
price = float(raw_price.replace(',', ''))
|
||||
|
||||
# Locate the element that contains the text 'In stock'
|
||||
# and filter out other elements.
|
||||
in_stock_element = context.page.locator(
|
||||
selector='span.product-form__inventory',
|
||||
has_text='In stock',
|
||||
).first
|
||||
in_stock = await in_stock_element.count() > 0
|
||||
|
||||
# Put it all together in a dictionary.
|
||||
data = {
|
||||
'manufacturer': manufacturer,
|
||||
'title': title,
|
||||
'sku': sku,
|
||||
'price': price,
|
||||
'in_stock': in_stock,
|
||||
}
|
||||
|
||||
# Print the extracted data.
|
||||
context.log.info(data)
|
||||
|
||||
# We are now on a category page. We can use this to paginate through and
|
||||
# enqueue all products, as well as any subsequent pages we find.
|
||||
elif context.request.label == 'CATEGORY':
|
||||
# Wait for the product items to render.
|
||||
await context.page.wait_for_selector('.product-item > a')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label DETAIL.
|
||||
await context.enqueue_links(
|
||||
selector='.product-item > a',
|
||||
label='DETAIL',
|
||||
)
|
||||
|
||||
# Find the "Next" button to paginate through the category pages.
|
||||
next_button = await context.page.query_selector('a.pagination__next')
|
||||
|
||||
# If a "Next" button is found, enqueue the next page of results.
|
||||
if next_button:
|
||||
await context.enqueue_links(
|
||||
selector='a.pagination__next',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
# This indicates we're on the start page with no specific label.
|
||||
# On the start page, we want to enqueue all the category pages.
|
||||
else:
|
||||
# Wait for the collection cards to render.
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label CATEGORY.
|
||||
await context.enqueue_links(
|
||||
selector='.collection-block-item',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,97 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Let's limit our crawls to make our tests shorter and safer.
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url}')
|
||||
|
||||
# We're not processing detail pages yet, so we just pass.
|
||||
if context.request.label == 'DETAIL':
|
||||
# Split the URL and get the last part to extract the manufacturer.
|
||||
url_part = context.request.url.split('/').pop()
|
||||
manufacturer = url_part.split('-')[0]
|
||||
|
||||
# Extract the title using the combined selector.
|
||||
title = await context.page.locator('.product-meta h1').text_content()
|
||||
|
||||
# Extract the SKU using its selector.
|
||||
sku = await context.page.locator(
|
||||
'span.product-meta__sku-number'
|
||||
).text_content()
|
||||
|
||||
# Locate the price element that contains the '$' sign and filter out
|
||||
# the visually hidden elements.
|
||||
price_element = context.page.locator('span.price', has_text='$').first
|
||||
current_price_string = await price_element.text_content() or ''
|
||||
raw_price = current_price_string.split('$')[1]
|
||||
price = float(raw_price.replace(',', ''))
|
||||
|
||||
# Locate the element that contains the text 'In stock' and filter out
|
||||
# other elements.
|
||||
in_stock_element = context.page.locator(
|
||||
selector='span.product-form__inventory',
|
||||
has_text='In stock',
|
||||
).first
|
||||
in_stock = await in_stock_element.count() > 0
|
||||
|
||||
# Put it all together in a dictionary.
|
||||
data = {
|
||||
'manufacturer': manufacturer,
|
||||
'title': title,
|
||||
'sku': sku,
|
||||
'price': price,
|
||||
'in_stock': in_stock,
|
||||
}
|
||||
|
||||
# Push the data to the dataset.
|
||||
await context.push_data(data)
|
||||
|
||||
# We are now on a category page. We can use this to paginate through and
|
||||
# enqueue all products, as well as any subsequent pages we find.
|
||||
elif context.request.label == 'CATEGORY':
|
||||
# Wait for the product items to render.
|
||||
await context.page.wait_for_selector('.product-item > a')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label DETAIL.
|
||||
await context.enqueue_links(
|
||||
selector='.product-item > a',
|
||||
label='DETAIL',
|
||||
)
|
||||
|
||||
# Find the "Next" button to paginate through the category pages.
|
||||
next_button = await context.page.query_selector('a.pagination__next')
|
||||
|
||||
# If a "Next" button is found, enqueue the next page of results.
|
||||
if next_button:
|
||||
await context.enqueue_links(
|
||||
selector='a.pagination__next',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
# This indicates we're on the start page with no specific label.
|
||||
# On the start page, we want to enqueue all the category pages.
|
||||
else:
|
||||
# Wait for the collection cards to render.
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
# Enqueue links found within elements matching the provided selector.
|
||||
# These links will be added to the crawling queue with the label CATEGORY.
|
||||
await context.enqueue_links(
|
||||
selector='.collection-block-item',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,22 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storages import Dataset
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler()
|
||||
dataset = await Dataset.open()
|
||||
|
||||
# ...
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
...
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
|
||||
from .routes import router
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Let's limit our crawls to make our tests shorter and safer.
|
||||
max_requests_per_crawl=10,
|
||||
# Provide our router instance to the crawler.
|
||||
request_handler=router,
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
from crawlee.crawlers import PlaywrightCrawlingContext
|
||||
from crawlee.router import Router
|
||||
|
||||
router = Router[PlaywrightCrawlingContext]()
|
||||
|
||||
|
||||
@router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# This is a fallback route which will handle the start URL.
|
||||
context.log.info(f'default_handler is processing {context.request.url}')
|
||||
|
||||
await context.page.wait_for_selector('.collection-block-item')
|
||||
|
||||
await context.enqueue_links(
|
||||
selector='.collection-block-item',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
|
||||
@router.handler('CATEGORY')
|
||||
async def category_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# This replaces the context.request.label == CATEGORY branch of the if clause.
|
||||
context.log.info(f'category_handler is processing {context.request.url}')
|
||||
|
||||
await context.page.wait_for_selector('.product-item > a')
|
||||
|
||||
await context.enqueue_links(
|
||||
selector='.product-item > a',
|
||||
label='DETAIL',
|
||||
)
|
||||
|
||||
next_button = await context.page.query_selector('a.pagination__next')
|
||||
|
||||
if next_button:
|
||||
await context.enqueue_links(
|
||||
selector='a.pagination__next',
|
||||
label='CATEGORY',
|
||||
)
|
||||
|
||||
|
||||
@router.handler('DETAIL')
|
||||
async def detail_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# This replaces the context.request.label == DETAIL branch of the if clause.
|
||||
context.log.info(f'detail_handler is processing {context.request.url}')
|
||||
|
||||
url_part = context.request.url.split('/').pop()
|
||||
manufacturer = url_part.split('-')[0]
|
||||
|
||||
title = await context.page.locator('.product-meta h1').text_content()
|
||||
|
||||
sku = await context.page.locator('span.product-meta__sku-number').text_content()
|
||||
|
||||
price_element = context.page.locator('span.price', has_text='$').first
|
||||
current_price_string = await price_element.text_content() or ''
|
||||
raw_price = current_price_string.split('$')[1]
|
||||
price = float(raw_price.replace(',', ''))
|
||||
|
||||
in_stock_element = context.page.locator(
|
||||
selector='span.product-form__inventory',
|
||||
has_text='In stock',
|
||||
).first
|
||||
in_stock = await in_stock_element.count() > 0
|
||||
|
||||
data = {
|
||||
'manufacturer': manufacturer,
|
||||
'title': title,
|
||||
'sku': sku,
|
||||
'price': price,
|
||||
'in_stock': in_stock,
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
@@ -0,0 +1,25 @@
|
||||
import asyncio
|
||||
|
||||
# highlight-next-line
|
||||
from apify import Actor
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
|
||||
from .routes import router
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# highlight-next-line
|
||||
async with Actor:
|
||||
crawler = PlaywrightCrawler(
|
||||
# Let's limit our crawls to make our tests shorter and safer.
|
||||
max_requests_per_crawl=10,
|
||||
# Provide our router instance to the crawler.
|
||||
request_handler=router,
|
||||
)
|
||||
|
||||
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,4 @@
|
||||
from crawlee.crawlers import PlaywrightCrawlingContext
|
||||
from crawlee.router import Router
|
||||
|
||||
router = Router[PlaywrightCrawlingContext]()
|
||||
Reference in New Issue
Block a user