同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
+30
View File
@@ -0,0 +1,30 @@
import asyncio
# Add import of crawler and crawling context.
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.storages import RequestQueue
async def main() -> None:
# First you create the request queue instance.
rq = await RequestQueue.open()
# And then you add one or more requests to it.
await rq.add_request('https://crawlee.dev')
crawler = BeautifulSoupCrawler(request_manager=rq)
# Define a request handler and attach it to the crawler using the decorator.
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
# Extract <title> text with BeautifulSoup.
# See BeautifulSoup documentation for API docs.
url = context.request.url
title = context.soup.title.string if context.soup.title else ''
context.log.info(f'The title of {url} is: {title}.')
await crawler.run()
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
# You don't need to import RequestQueue anymore.
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
crawler = BeautifulSoupCrawler()
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
url = context.request.url
title = context.soup.title.string if context.soup.title else ''
context.log.info(f'The title of {url} is: {title}.')
# Start the crawler with the provided URLs.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,15 @@
import asyncio
from crawlee.storages import RequestQueue
async def main() -> None:
# First you create the request queue instance.
rq = await RequestQueue.open()
# And then you add one or more requests to it.
await rq.add_request('https://crawlee.dev')
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,25 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}.')
# See the `EnqueueStrategy` type alias for more strategy options.
# highlight-next-line
await context.enqueue_links(
# highlight-next-line
strategy='same-domain',
# highlight-next-line
)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,24 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
# Let's limit our crawls to make our tests shorter and safer.
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
url = context.request.url
title = context.soup.title.string if context.soup.title else ''
context.log.info(f'The title of {url} is: {title}.')
# The enqueue_links function is available as one of the fields of the context.
# It is also context aware, so it does not require any parameters.
await context.enqueue_links()
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,29 @@
import asyncio
from crawlee import Glob
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}.')
# Enqueue links that match the 'include' glob pattern and
# do not match the 'exclude' glob pattern.
# highlight-next-line
await context.enqueue_links(
# highlight-next-line
include=[Glob('https://someplace.com/**/cats')],
# highlight-next-line
exclude=[Glob('https://**/archive/**')],
# highlight-next-line
)
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,19 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
async def main() -> None:
crawler = BeautifulSoupCrawler()
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
url = context.request.url
title = context.soup.title.string if context.soup.title else ''
context.log.info(f'The title of {url} is: {title}.')
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,49 @@
from __future__ import annotations
import asyncio
from crawlee import HttpHeaders, RequestOptions, RequestTransformAction
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
def transform_request(
request_options: RequestOptions,
) -> RequestOptions | RequestTransformAction:
# Skip requests to PDF files
if request_options['url'].endswith('.pdf'):
return 'skip'
if '/docs' in request_options['url']:
# Add custom headers to requests to specific URLs
request_options['headers'] = HttpHeaders({'Custom-Header': 'value'})
elif '/blog' in request_options['url']:
# Add label for certain URLs
request_options['label'] = 'BLOG'
else:
# Signal that the request should proceed without any transformation
return 'unchanged'
return request_options
async def main() -> None:
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}.')
# Transform request before enqueueing
await context.enqueue_links(transform_request_function=transform_request)
@crawler.router.handler('BLOG')
async def blog_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Blog Processing {context.request.url}.')
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,32 @@
import asyncio
# Instead of BeautifulSoupCrawler let's use Playwright to be able to render JavaScript.
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler()
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
# Wait for the collection cards to render on the page. This ensures that
# the elements we want to interact with are present in the DOM.
await context.page.wait_for_selector('.collection-block-item')
# Execute a function within the browser context to target the collection
# card elements and extract their text content, trimming any leading or
# trailing whitespace.
category_texts = await context.page.eval_on_selector_all(
'.collection-block-item',
'(els) => els.map(el => el.textContent.trim())',
)
# Log the extracted texts.
for i, text in enumerate(category_texts):
context.log.info(f'CATEGORY_{i + 1}: {text}')
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,57 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler()
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# We're not processing detail pages yet, so we just pass.
if context.request.label == 'DETAIL':
pass
# We are now on a category page. We can use this to paginate through and
# enqueue all products, as well as any subsequent pages we find.
elif context.request.label == 'CATEGORY':
# Wait for the product items to render.
await context.page.wait_for_selector('.product-item > a')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label DETAIL.
await context.enqueue_links(
selector='.product-item > a',
label='DETAIL',
)
# Find the "Next" button to paginate through the category pages.
next_button = await context.page.query_selector('a.pagination__next')
# If a "Next" button is found, enqueue the next page of results.
if next_button:
await context.enqueue_links(
selector='a.pagination__next',
label='CATEGORY',
)
# This indicates we're on the start page with no specific label.
# On the start page, we want to enqueue all the category pages.
else:
# Wait for the collection cards to render.
await context.page.wait_for_selector('.collection-block-item')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label CATEGORY.
await context.enqueue_links(
selector='.collection-block-item',
label='CATEGORY',
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,28 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler()
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# Wait for the category cards to render on the page. This ensures that
# the elements we want to interact with are present in the DOM.
await context.page.wait_for_selector('.collection-block-item')
# Enqueue links found within elements that match the specified selector.
# These links will be added to the crawling queue with the label CATEGORY.
await context.enqueue_links(
selector='.collection-block-item',
label='CATEGORY',
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,97 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler(
# Let's limit our crawls to make our tests shorter and safer.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# We're not processing detail pages yet, so we just pass.
if context.request.label == 'DETAIL':
# Split the URL and get the last part to extract the manufacturer.
url_part = context.request.url.split('/').pop()
manufacturer = url_part.split('-')[0]
# Extract the title using the combined selector.
title = await context.page.locator('.product-meta h1').text_content()
# Extract the SKU using its selector.
sku = await context.page.locator(
'span.product-meta__sku-number'
).text_content()
# Locate the price element that contains the '$' sign and filter out
# the visually hidden elements.
price_element = context.page.locator('span.price', has_text='$').first
current_price_string = await price_element.text_content() or ''
raw_price = current_price_string.split('$')[1]
price = float(raw_price.replace(',', ''))
# Locate the element that contains the text 'In stock'
# and filter out other elements.
in_stock_element = context.page.locator(
selector='span.product-form__inventory',
has_text='In stock',
).first
in_stock = await in_stock_element.count() > 0
# Put it all together in a dictionary.
data = {
'manufacturer': manufacturer,
'title': title,
'sku': sku,
'price': price,
'in_stock': in_stock,
}
# Print the extracted data.
context.log.info(data)
# We are now on a category page. We can use this to paginate through and
# enqueue all products, as well as any subsequent pages we find.
elif context.request.label == 'CATEGORY':
# Wait for the product items to render.
await context.page.wait_for_selector('.product-item > a')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label DETAIL.
await context.enqueue_links(
selector='.product-item > a',
label='DETAIL',
)
# Find the "Next" button to paginate through the category pages.
next_button = await context.page.query_selector('a.pagination__next')
# If a "Next" button is found, enqueue the next page of results.
if next_button:
await context.enqueue_links(
selector='a.pagination__next',
label='CATEGORY',
)
# This indicates we're on the start page with no specific label.
# On the start page, we want to enqueue all the category pages.
else:
# Wait for the collection cards to render.
await context.page.wait_for_selector('.collection-block-item')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label CATEGORY.
await context.enqueue_links(
selector='.collection-block-item',
label='CATEGORY',
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,97 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
async def main() -> None:
crawler = PlaywrightCrawler(
# Let's limit our crawls to make our tests shorter and safer.
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url}')
# We're not processing detail pages yet, so we just pass.
if context.request.label == 'DETAIL':
# Split the URL and get the last part to extract the manufacturer.
url_part = context.request.url.split('/').pop()
manufacturer = url_part.split('-')[0]
# Extract the title using the combined selector.
title = await context.page.locator('.product-meta h1').text_content()
# Extract the SKU using its selector.
sku = await context.page.locator(
'span.product-meta__sku-number'
).text_content()
# Locate the price element that contains the '$' sign and filter out
# the visually hidden elements.
price_element = context.page.locator('span.price', has_text='$').first
current_price_string = await price_element.text_content() or ''
raw_price = current_price_string.split('$')[1]
price = float(raw_price.replace(',', ''))
# Locate the element that contains the text 'In stock' and filter out
# other elements.
in_stock_element = context.page.locator(
selector='span.product-form__inventory',
has_text='In stock',
).first
in_stock = await in_stock_element.count() > 0
# Put it all together in a dictionary.
data = {
'manufacturer': manufacturer,
'title': title,
'sku': sku,
'price': price,
'in_stock': in_stock,
}
# Push the data to the dataset.
await context.push_data(data)
# We are now on a category page. We can use this to paginate through and
# enqueue all products, as well as any subsequent pages we find.
elif context.request.label == 'CATEGORY':
# Wait for the product items to render.
await context.page.wait_for_selector('.product-item > a')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label DETAIL.
await context.enqueue_links(
selector='.product-item > a',
label='DETAIL',
)
# Find the "Next" button to paginate through the category pages.
next_button = await context.page.query_selector('a.pagination__next')
# If a "Next" button is found, enqueue the next page of results.
if next_button:
await context.enqueue_links(
selector='a.pagination__next',
label='CATEGORY',
)
# This indicates we're on the start page with no specific label.
# On the start page, we want to enqueue all the category pages.
else:
# Wait for the collection cards to render.
await context.page.wait_for_selector('.collection-block-item')
# Enqueue links found within elements matching the provided selector.
# These links will be added to the crawling queue with the label CATEGORY.
await context.enqueue_links(
selector='.collection-block-item',
label='CATEGORY',
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,22 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.storages import Dataset
# ...
async def main() -> None:
crawler = PlaywrightCrawler()
dataset = await Dataset.open()
# ...
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
...
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,20 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler
from .routes import router
async def main() -> None:
crawler = PlaywrightCrawler(
# Let's limit our crawls to make our tests shorter and safer.
max_requests_per_crawl=10,
# Provide our router instance to the crawler.
request_handler=router,
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,72 @@
from crawlee.crawlers import PlaywrightCrawlingContext
from crawlee.router import Router
router = Router[PlaywrightCrawlingContext]()
@router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# This is a fallback route which will handle the start URL.
context.log.info(f'default_handler is processing {context.request.url}')
await context.page.wait_for_selector('.collection-block-item')
await context.enqueue_links(
selector='.collection-block-item',
label='CATEGORY',
)
@router.handler('CATEGORY')
async def category_handler(context: PlaywrightCrawlingContext) -> None:
# This replaces the context.request.label == CATEGORY branch of the if clause.
context.log.info(f'category_handler is processing {context.request.url}')
await context.page.wait_for_selector('.product-item > a')
await context.enqueue_links(
selector='.product-item > a',
label='DETAIL',
)
next_button = await context.page.query_selector('a.pagination__next')
if next_button:
await context.enqueue_links(
selector='a.pagination__next',
label='CATEGORY',
)
@router.handler('DETAIL')
async def detail_handler(context: PlaywrightCrawlingContext) -> None:
# This replaces the context.request.label == DETAIL branch of the if clause.
context.log.info(f'detail_handler is processing {context.request.url}')
url_part = context.request.url.split('/').pop()
manufacturer = url_part.split('-')[0]
title = await context.page.locator('.product-meta h1').text_content()
sku = await context.page.locator('span.product-meta__sku-number').text_content()
price_element = context.page.locator('span.price', has_text='$').first
current_price_string = await price_element.text_content() or ''
raw_price = current_price_string.split('$')[1]
price = float(raw_price.replace(',', ''))
in_stock_element = context.page.locator(
selector='span.product-form__inventory',
has_text='In stock',
).first
in_stock = await in_stock_element.count() > 0
data = {
'manufacturer': manufacturer,
'title': title,
'sku': sku,
'price': price,
'in_stock': in_stock,
}
await context.push_data(data)
@@ -0,0 +1,25 @@
import asyncio
# highlight-next-line
from apify import Actor
from crawlee.crawlers import PlaywrightCrawler
from .routes import router
async def main() -> None:
# highlight-next-line
async with Actor:
crawler = PlaywrightCrawler(
# Let's limit our crawls to make our tests shorter and safer.
max_requests_per_crawl=10,
# Provide our router instance to the crawler.
request_handler=router,
)
await crawler.run(['https://warehouse-theme-metal.myshopify.com/collections'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,4 @@
from crawlee.crawlers import PlaywrightCrawlingContext
from crawlee.router import Router
router = Router[PlaywrightCrawlingContext]()