同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Wrap the crawler code in an Actor context manager.
|
||||
async with Actor:
|
||||
crawler = BeautifulSoupCrawler(max_requests_per_crawl=10)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
}
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,16 @@
|
||||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
store = await Actor.open_key_value_store()
|
||||
await store.set_value('your-file', {'foo': 'bar'})
|
||||
url = store.get_public_url('your-file')
|
||||
Actor.log.info(f'KVS public URL: {url}')
|
||||
# https://api.apify.com/v2/key-value-stores/<your-store-id>/records/your-file
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,19 @@
|
||||
import asyncio
|
||||
|
||||
from apify import Actor, Configuration
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
# Create a new configuration with your API key. You can find it at
|
||||
# https://console.apify.com/settings/integrations. It can be provided either
|
||||
# as a parameter "token" or as an environment variable "APIFY_TOKEN".
|
||||
config = Configuration(
|
||||
token='apify_api_YOUR_TOKEN',
|
||||
)
|
||||
|
||||
async with Actor(config):
|
||||
Actor.log.info('Hello from Apify platform!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,20 @@
|
||||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
proxy_configuration = await Actor.create_proxy_configuration(
|
||||
password='apify_proxy_YOUR_PASSWORD',
|
||||
# Specify the proxy group to use.
|
||||
groups=['RESIDENTIAL'],
|
||||
# Set the country code for the proxy.
|
||||
country_code='US',
|
||||
)
|
||||
|
||||
# ...
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,24 @@
|
||||
import asyncio
|
||||
|
||||
from apify import Actor
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
async with Actor:
|
||||
# Create a new Apify Proxy configuration. The password can be found at
|
||||
# https://console.apify.com/proxy/http-settings and should be provided either
|
||||
# as a parameter "password" or as an environment variable "APIFY_PROXY_PASSWORD".
|
||||
proxy_configuration = await Actor.create_proxy_configuration(
|
||||
password='apify_proxy_YOUR_PASSWORD',
|
||||
)
|
||||
|
||||
if not proxy_configuration:
|
||||
Actor.log.warning('Failed to create proxy configuration.')
|
||||
return
|
||||
|
||||
proxy_url = await proxy_configuration.new_url()
|
||||
Actor.log.info(f'Proxy URL: {proxy_url}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user