同步完整源码 - 2026-05-24
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
import json
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
from litestar import Litestar, get
|
||||
|
||||
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
@get('/')
|
||||
async def main() -> str:
|
||||
"""The crawler entry point that will be called when the HTTP endpoint is accessed."""
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
crawler = PlaywrightCrawler(
|
||||
headless=True,
|
||||
max_requests_per_crawl=10,
|
||||
browser_type='firefox',
|
||||
storage_client=storage_client,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def default_handler(context: PlaywrightCrawlingContext) -> None:
|
||||
"""Default request handler that processes each page during crawling."""
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
title = await context.page.query_selector('title')
|
||||
await context.push_data(
|
||||
{
|
||||
'url': context.request.loaded_url,
|
||||
'title': await title.inner_text() if title else None,
|
||||
}
|
||||
)
|
||||
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
data = await crawler.get_data()
|
||||
|
||||
# Return the results as JSON to the client
|
||||
return json.dumps(data.items)
|
||||
|
||||
|
||||
# Initialize the Litestar app with our route handler
|
||||
app = Litestar(route_handlers=[main])
|
||||
|
||||
# Start the Uvicorn server using the `PORT` environment variable provided by GCP
|
||||
# This is crucial - Cloud Run expects your app to listen on this specific port
|
||||
uvicorn.run(app, host='0.0.0.0', port=int(os.environ.get('PORT', '8080'))) # noqa: S104 # Use all interfaces in a container, safely
|
||||
@@ -0,0 +1,57 @@
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import timedelta
|
||||
|
||||
import functions_framework
|
||||
from flask import Request, Response
|
||||
|
||||
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
|
||||
from crawlee.storage_clients import MemoryStorageClient
|
||||
|
||||
|
||||
async def main() -> str:
|
||||
# highlight-start
|
||||
# Disable writing storage data to the file system
|
||||
storage_client = MemoryStorageClient()
|
||||
# highlight-end
|
||||
|
||||
crawler = BeautifulSoupCrawler(
|
||||
storage_client=storage_client,
|
||||
max_request_retries=1,
|
||||
request_handler_timeout=timedelta(seconds=30),
|
||||
max_requests_per_crawl=10,
|
||||
)
|
||||
|
||||
@crawler.router.default_handler
|
||||
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
|
||||
context.log.info(f'Processing {context.request.url} ...')
|
||||
|
||||
data = {
|
||||
'url': context.request.url,
|
||||
'title': context.soup.title.string if context.soup.title else None,
|
||||
'h1s': [h1.text for h1 in context.soup.find_all('h1')],
|
||||
'h2s': [h2.text for h2 in context.soup.find_all('h2')],
|
||||
'h3s': [h3.text for h3 in context.soup.find_all('h3')],
|
||||
}
|
||||
|
||||
await context.push_data(data)
|
||||
await context.enqueue_links()
|
||||
|
||||
await crawler.run(['https://crawlee.dev'])
|
||||
|
||||
# highlight-start
|
||||
# Extract data saved in `Dataset`
|
||||
data = await crawler.get_data()
|
||||
# Serialize to json string and return
|
||||
return json.dumps(data.items)
|
||||
# highlight-end
|
||||
|
||||
|
||||
@functions_framework.http
|
||||
def crawlee_run(request: Request) -> Response:
|
||||
# You can pass data to your crawler using `request`
|
||||
function_id = request.headers['Function-Execution-Id']
|
||||
response_str = asyncio.run(main())
|
||||
|
||||
# Return a response with the crawling results
|
||||
return Response(response=response_str, status=200)
|
||||
Reference in New Issue
Block a user