同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import CurlImpersonateHttpClient
async def main() -> None:
http_client = CurlImpersonateHttpClient(
# Optional additional keyword arguments for `curl_cffi.requests.AsyncSession`.
timeout=10,
impersonate='chrome131',
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,42 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import HttpxHttpClient
async def main() -> None:
http_client = HttpxHttpClient(
# Optional additional keyword arguments for `httpx.AsyncClient`.
timeout=10,
follow_redirects=True,
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,43 @@
import asyncio
from crawlee.crawlers import ParselCrawler, ParselCrawlingContext
from crawlee.http_clients import ImpitHttpClient
async def main() -> None:
http_client = ImpitHttpClient(
# Optional additional keyword arguments for `impit.AsyncClient`.
http3=True,
browser='firefox',
verify=True,
)
crawler = ParselCrawler(
http_client=http_client,
# Limit the crawl to max requests. Remove or increase it for crawling all links.
max_requests_per_crawl=10,
)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def request_handler(context: ParselCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
# Enqueue all links from the page.
await context.enqueue_links()
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.selector.css('title::text').get(),
}
# Push the extracted data to the default dataset.
await context.push_data(data)
# Run the crawler with the initial list of URLs.
await crawler.run(['https://crawlee.dev'])
if __name__ == '__main__':
asyncio.run(main())