Files
crawlee-python/tests/e2e/project_template/test_static_crawlers_templates.py

165 lines
6.7 KiB
Python

import os
import re
import subprocess
from pathlib import Path
from typing import Literal
import pytest
from apify_client import ApifyClientAsync
from cookiecutter.main import cookiecutter
from crawlee._cli import default_start_url, template_directory
from crawlee._utils.crypto import crypto_random_object_id
from tests.e2e.project_template.utils import patch_crawlee_version_in_project
# To run these tests locally, make sure you have apify-cli installed and available in the path.
# https://docs.apify.com/cli/docs/installation
@pytest.mark.parametrize(
'crawler_type',
[
pytest.param('playwright-camoufox', marks=pytest.mark.playwright_camoufox),
pytest.param('playwright-chrome', marks=pytest.mark.playwright_chrome),
pytest.param('playwright-firefox', marks=pytest.mark.playwright_firefox),
pytest.param('playwright-webkit', marks=pytest.mark.playwright_webkit),
pytest.param('playwright', marks=pytest.mark.playwright),
pytest.param('parsel', marks=pytest.mark.parsel),
pytest.param('beautifulsoup', marks=pytest.mark.beautifulsoup),
pytest.param('adaptive-beautifulsoup', marks=pytest.mark.adaptive_beautifulsoup),
pytest.param('adaptive-parsel', marks=pytest.mark.adaptive_parsel),
pytest.param('stagehand', marks=pytest.mark.stagehand),
],
)
@pytest.mark.parametrize(
'http_client',
[
pytest.param('httpx', marks=pytest.mark.httpx),
pytest.param('curl-impersonate', marks=pytest.mark.curl_impersonate),
pytest.param('impit', marks=pytest.mark.impit),
],
)
@pytest.mark.parametrize(
'package_manager',
[
pytest.param('pip', marks=pytest.mark.pip),
pytest.param('uv', marks=pytest.mark.uv),
pytest.param('poetry', marks=pytest.mark.poetry),
],
)
@pytest.mark.flaky(
reruns=3,
reason='`apify push` occasionally hangs against the Apify platform; retries cover transient CLI/network flakes.',
)
async def test_static_crawler_actor_at_apify(
tmp_path: Path,
crawlee_wheel_path: Path,
package_manager: Literal['pip', 'uv', 'poetry'],
crawler_type: str,
http_client: str,
) -> None:
# Generate new actor name
actor_name = f'crawlee-python-template-e2e-test-{crypto_random_object_id(8).lower()}'
# Create project from template
cookiecutter(
template=str(template_directory),
no_input=True,
extra_context={
'project_name': actor_name,
'package_manager': package_manager,
'crawler_type': crawler_type,
'http_client': http_client,
'enable_apify_integration': True,
'start_url': default_start_url,
'install_project': False,
},
output_dir=str(tmp_path),
)
patch_crawlee_version_in_project(
project_path=tmp_path / actor_name, wheel_path=crawlee_wheel_path, package_manager=package_manager
)
# Print apify version for debugging purposes in rare cases of CLI failures
subprocess.run(['apify', '--version'], check=True) # noqa: ASYNC221, S607
# Build actor using sequence of cli commands as the user would.
subprocess.run( # noqa: ASYNC221, S603
['apify', 'login', '-t', os.environ['APIFY_TEST_USER_API_TOKEN']], # noqa: S607
capture_output=True,
check=True,
cwd=tmp_path / actor_name,
timeout=600,
)
subprocess.run( # noqa: ASYNC221, S603
['apify', 'init', '-y', actor_name], # noqa: S607
capture_output=True,
check=True,
cwd=tmp_path / actor_name,
timeout=600,
)
build_process = subprocess.run( # noqa: ASYNC221
['apify', 'push'], # noqa: S607
capture_output=True,
check=False,
cwd=tmp_path / actor_name,
# Prevent git from walking up into the surrounding project's .git/ when running under
# a basetemp inside the repo (see --basetemp in the e2e-templates-tests poe task).
env={**os.environ, 'GIT_CEILING_DIRECTORIES': str(tmp_path)},
timeout=600,
)
# Get actor ID from build log
actor_id_regexp = re.compile(r'https:\/\/console\.apify\.com\/actors\/(.*)#\/builds\/\d*\.\d*\.\d*')
if match := re.findall(actor_id_regexp, build_process.stderr.decode()):
actor_id = match[0]
else:
raise AssertionError(f'Failed to find actor id in build log: {build_process.stderr.decode()}')
client = ApifyClientAsync(token=os.getenv('APIFY_TEST_USER_API_TOKEN'))
actor = client.actor(actor_id)
# Run actor
try:
assert build_process.returncode == 0
# Stagehand needs the model API key in `os.environ` at run time. Env vars are baked into the
# build at build creation time, so registering the env var on the version after `apify push`
# is not enough — we must trigger a fresh build for the env var to be included.
build_number: str | None = None
if crawler_type == 'stagehand':
env_vars = actor.version('0.0').env_vars()
await env_vars.create(name='OPENAI_API_KEY', value=os.environ['OPENAI_API_KEY'], is_secret=True)
# `ActorClientAsync.build`'s `wait_for_finish` parameter is capped server-side at 60s,
# which is shorter than a stagehand build (playwright + browser deps). Trigger the build,
# then poll client-side via `BuildClientAsync.wait_for_finish` until it reaches a terminal
# status, and assert it succeeded before starting the run.
rebuild = await actor.build(version_number='0.0')
finished_build = await client.build(rebuild['id']).wait_for_finish(wait_secs=900)
assert finished_build is not None, 'Stagehand rebuild did not reach a terminal status within 900s.'
assert finished_build['status'] == 'SUCCEEDED', (
f'Stagehand rebuild did not succeed: status={finished_build["status"]!r}, build={finished_build}'
)
build_number = finished_build['buildNumber']
started_run_data = await actor.start(memory_mbytes=8192, build=build_number)
actor_run = client.run(started_run_data['id'])
finished_run_data = await actor_run.wait_for_finish()
actor_run_log = await actor_run.log().get()
finally:
# Delete the actor once it is no longer needed.
await actor.delete()
# Asserts
additional_run_info = f'Full actor run log: {actor_run_log}'
assert actor_run_log
assert finished_run_data
assert finished_run_data['status'] == 'SUCCEEDED', additional_run_info
assert (
'Crawler.stop() was called with following reason: The crawler has reached its limit of 10 requests per crawl.'
) in actor_run_log, additional_run_info
assert int(re.findall(r'requests_finished\s*│\s*(\d*)', actor_run_log)[-1]) >= 10, additional_run_info