同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,28 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
context.log.info(f'Extracted data: {data}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Extract data from the page.
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': await context.page.title(),
|
||||
}
|
||||
context.log.info(f'Extracted data: {data}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,21 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
|
||||
# The proxy URLs are rotated in a round-robin.
|
||||
proxy_url_1 = await proxy_configuration.new_url() # http://proxy-1.com/
|
||||
proxy_url_2 = await proxy_configuration.new_url() # http://proxy-2.com/
|
||||
proxy_url_3 = await proxy_configuration.new_url() # http://proxy-1.com/
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
use_session_pool=True,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
proxy_urls=[
|
||||
'http://proxy-1.com/',
|
||||
'http://proxy-2.com/',
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(
|
||||
proxy_configuration=proxy_configuration,
|
||||
use_session_pool=True,
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
tiered_proxy_urls=[
|
||||
# No proxy tier.
|
||||
# Optional in case you do not want to use any proxy on lowest tier.
|
||||
[None],
|
||||
# lower tier, cheaper, preferred as long as they work
|
||||
[
|
||||
'http://cheap-datacenter-proxy-1.com/',
|
||||
'http://cheap-datacenter-proxy-2.com/',
|
||||
],
|
||||
# higher tier, more expensive, used as a fallback
|
||||
[
|
||||
'http://expensive-residential-proxy-1.com/',
|
||||
'http://expensive-residential-proxy-2.com/',
|
||||
],
|
||||
]
|
||||
)
|
||||
crawler = BeautifulSoupCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,39 @@
|
||||
import asyncio
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.proxy_configuration import ProxyConfiguration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a ProxyConfiguration object and pass it to the crawler.
|
||||
proxy_configuration = ProxyConfiguration(
|
||||
tiered_proxy_urls=[
|
||||
# No proxy tier.
|
||||
# Optional in case you do not want to use any proxy on lowest tier.
|
||||
[None],
|
||||
# lower tier, cheaper, preferred as long as they work
|
||||
[
|
||||
'http://cheap-datacenter-proxy-1.com/',
|
||||
'http://cheap-datacenter-proxy-2.com/',
|
||||
],
|
||||
# higher tier, more expensive, used as a fallback
|
||||
[
|
||||
'http://expensive-residential-proxy-1.com/',
|
||||
'http://expensive-residential-proxy-2.com/',
|
||||
],
|
||||
]
|
||||
)
|
||||
crawler = PlaywrightCrawler(proxy_configuration=proxy_configuration)
|
||||
|
||||
# Define the default request handler, which will be called for every request.
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
# Log the proxy used for the current request.
|
||||
context.log.info(f'Proxy for the current request: {context.proxy_info}')
|
||||
|
||||
# Run the crawler with the initial list of requests.
|
||||
await crawler.run(['https://crawlee.dev/'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user