同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
@@ -0,0 +1,28 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,28 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,32 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Extract data from the page.
data = {
'url': context.request.url,
'title': context.soup.title.string if context.soup.title else None,
}
context.log.info(f'Extracted data: {data}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,32 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Extract data from the page.
data = {
'url': context.request.url,
'title': await context.page.title(),
}
context.log.info(f'Extracted data: {data}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,21 @@
import asyncio
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
# The proxy URLs are rotated in a round-robin.
proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/
proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/
proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,24 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = BeautifulSoupCrawler(
proxy_configuration=proxy_configuration,
use_session_pool=True,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,24 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
proxy_urls=[
'http://proxy-1.com/',
'http://proxy-2.com/',
]
)
crawler = PlaywrightCrawler(
proxy_configuration=proxy_configuration,
use_session_pool=True,
)
# ...
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,39 @@
import asyncio
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
tiered_proxy_urls=[
# No proxy tier.
# Optional in case you do not want to use any proxy on lowest tier.
[None],
# lower tier, cheaper, preferred as long as they work
[
'http://cheap-datacenter-proxy-1.com/',
'http://cheap-datacenter-proxy-2.com/',
],
# higher tier, more expensive, used as a fallback
[
'http://expensive-residential-proxy-1.com/',
'http://expensive-residential-proxy-2.com/',
],
]
)
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())
@@ -0,0 +1,39 @@
import asyncio
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.proxy_configuration import ProxyConfiguration
async def main() -> None:
# Create a ProxyConfiguration object and pass it to the crawler.
proxy_configuration = ProxyConfiguration(
tiered_proxy_urls=[
# No proxy tier.
# Optional in case you do not want to use any proxy on lowest tier.
[None],
# lower tier, cheaper, preferred as long as they work
[
'http://cheap-datacenter-proxy-1.com/',
'http://cheap-datacenter-proxy-2.com/',
],
# higher tier, more expensive, used as a fallback
[
'http://expensive-residential-proxy-1.com/',
'http://expensive-residential-proxy-2.com/',
],
]
)
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
# Define the default request handler, which will be called for every request.
@crawler.router.default_handler
async def default_handler(context: PlaywrightCrawlingContext) -> None:
# Log the proxy used for the current request.
context.log.info(f'Proxy for the current request: {context.proxy_info}')
# Run the crawler with the initial list of requests.
await crawler.run(['https://crawlee.dev/'])
if __name__ == '__main__':
asyncio.run(main())