同步完整源码 - 2026-05-24

This commit is contained in:
xiaoxue
2026-05-24 23:06:32 +08:00
commit 2b4682de8e
764 changed files with 98522 additions and 0 deletions
@@ -0,0 +1,61 @@
import asyncio
import json
from datetime import timedelta
from typing import Any
from aws_lambda_powertools.utilities.typing import LambdaContext
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
from crawlee.storage_clients import MemoryStorageClient
from crawlee.storages import Dataset, RequestQueue
async def main() -> str:
# highlight-start
# Disable writing storage data to the file system
storage_client = MemoryStorageClient()
# highlight-end
# Initialize storages
dataset = await Dataset.open(storage_client=storage_client)
request_queue = await RequestQueue.open(storage_client=storage_client)
crawler = BeautifulSoupCrawler(
storage_client=storage_client,
max_request_retries=1,
request_handler_timeout=timedelta(seconds=30),
max_requests_per_crawl=10,
)
@crawler.router.default_handler
async def request_handler(context: BeautifulSoupCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
data = {
'url': context.request.url,
'title': context.soup.title.string if context.soup.title else None,
'h1s': [h1.text for h1 in context.soup.find_all('h1')],
'h2s': [h2.text for h2 in context.soup.find_all('h2')],
'h3s': [h3.text for h3 in context.soup.find_all('h3')],
}
await context.push_data(data)
await context.enqueue_links()
await crawler.run(['https://crawlee.dev'])
# Extract data saved in `Dataset`
data = await crawler.get_data()
# Clean up storages after the crawl
await dataset.drop()
await request_queue.drop()
# Serialize the list of scraped items to JSON string
return json.dumps(data.items)
def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> dict[str, Any]:
result = asyncio.run(main())
# Return the response with results
return {'statusCode': 200, 'body': result}
@@ -0,0 +1,73 @@
import asyncio
import json
from datetime import timedelta
from typing import Any
from aws_lambda_powertools.utilities.typing import LambdaContext
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
from crawlee.storage_clients import MemoryStorageClient
from crawlee.storages import Dataset, RequestQueue
async def main() -> str:
# highlight-start
# Disable writing storage data to the file system
storage_client = MemoryStorageClient()
# highlight-end
# Initialize storages
dataset = await Dataset.open(storage_client=storage_client)
request_queue = await RequestQueue.open(storage_client=storage_client)
crawler = PlaywrightCrawler(
storage_client=storage_client,
max_request_retries=1,
request_handler_timeout=timedelta(seconds=30),
max_requests_per_crawl=10,
# highlight-start
# Configure Playwright to run in AWS Lambda environment
browser_launch_options={
'args': [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
'--single-process',
]
},
# highlight-end
)
@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
context.log.info(f'Processing {context.request.url} ...')
data = {
'url': context.request.url,
'title': await context.page.title(),
'h1s': await context.page.locator('h1').all_text_contents(),
'h2s': await context.page.locator('h2').all_text_contents(),
'h3s': await context.page.locator('h3').all_text_contents(),
}
await context.push_data(data)
await context.enqueue_links()
await crawler.run(['https://crawlee.dev'])
# Extract data saved in `Dataset`
data = await crawler.get_data()
# Clean up storages after the crawl
await dataset.drop()
await request_queue.drop()
# Serialize the list of scraped items to JSON string
return json.dumps(data.items)
def lambda_handler(_event: dict[str, Any], _context: LambdaContext) -> dict[str, Any]:
result = asyncio.run(main())
# Return the response with results
return {'statusCode': 200, 'body': result}
@@ -0,0 +1,36 @@
FROM apify/actor-python-playwright:3.14
RUN apt update && apt install -yq git && rm -rf /var/lib/apt/lists/*
RUN pip install -U pip setuptools \
&& pip install 'uv<1'
ENV UV_PROJECT_ENVIRONMENT="/usr/local"
COPY pyproject.toml uv.lock ./
RUN echo "Python version:" \
&& python --version \
&& echo "Installing dependencies:" \
&& PLAYWRIGHT_INSTALLED=$(pip freeze | grep -q playwright && echo "true" || echo "false") \
&& if [ "$PLAYWRIGHT_INSTALLED" = "true" ]; then \
echo "Playwright already installed, excluding from uv sync" \
&& uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact --no-install-package playwright; \
else \
echo "Playwright not found, installing all dependencies" \
&& uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact; \
fi \
&& echo "All installed Python packages:" \
&& pip freeze
COPY . ./
RUN python -m compileall -q .
# highlight-start
# AWS Lambda entrypoint
ENTRYPOINT [ "/usr/local/bin/python3", "-m", "awslambdaric" ]
# Lambda handler function
CMD [ "aws_playwright.main.lambda_handler" ]
# highlight-end